Create benchmark

curl --request POST \
  --url https://api.mixpeek.com/v1/retrievers/benchmarks \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "benchmark_name": "<string>",
  "baseline_retriever_id": "<string>",
  "candidate_retriever_ids": [
    "<string>"
  ],
  "session_filter": {
    "retriever_ids": [
      "<string>"
    ],
    "taxonomy_node_ids": [
      "<string>"
    ],
    "time_range": {
      "start": "2023-11-07T05:31:56Z",
      "end": "2023-11-07T05:31:56Z"
    },
    "min_interactions": 1,
    "interaction_types": [
      "<string>"
    ],
    "sample_strategy": "random",
    "interaction_weights": {
      "weights": {}
    }
  },
  "session_count": 1000
}
'

import requests

url = "https://api.mixpeek.com/v1/retrievers/benchmarks"

payload = {
    "benchmark_name": "<string>",
    "baseline_retriever_id": "<string>",
    "candidate_retriever_ids": ["<string>"],
    "session_filter": {
        "retriever_ids": ["<string>"],
        "taxonomy_node_ids": ["<string>"],
        "time_range": {
            "start": "2023-11-07T05:31:56Z",
            "end": "2023-11-07T05:31:56Z"
        },
        "min_interactions": 1,
        "interaction_types": ["<string>"],
        "sample_strategy": "random",
        "interaction_weights": { "weights": {} }
    },
    "session_count": 1000
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    benchmark_name: '<string>',
    baseline_retriever_id: '<string>',
    candidate_retriever_ids: ['<string>'],
    session_filter: {
      retriever_ids: ['<string>'],
      taxonomy_node_ids: ['<string>'],
      time_range: {start: '2023-11-07T05:31:56Z', end: '2023-11-07T05:31:56Z'},
      min_interactions: 1,
      interaction_types: ['<string>'],
      sample_strategy: 'random',
      interaction_weights: {weights: {}}
    },
    session_count: 1000
  })
};

fetch('https://api.mixpeek.com/v1/retrievers/benchmarks', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.mixpeek.com/v1/retrievers/benchmarks",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'benchmark_name' => '<string>',
    'baseline_retriever_id' => '<string>',
    'candidate_retriever_ids' => [
        '<string>'
    ],
    'session_filter' => [
        'retriever_ids' => [
                '<string>'
        ],
        'taxonomy_node_ids' => [
                '<string>'
        ],
        'time_range' => [
                'start' => '2023-11-07T05:31:56Z',
                'end' => '2023-11-07T05:31:56Z'
        ],
        'min_interactions' => 1,
        'interaction_types' => [
                '<string>'
        ],
        'sample_strategy' => 'random',
        'interaction_weights' => [
                'weights' => [
                                
                ]
        ]
    ],
    'session_count' => 1000
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.mixpeek.com/v1/retrievers/benchmarks"

	payload := strings.NewReader("{\n  \"benchmark_name\": \"<string>\",\n  \"baseline_retriever_id\": \"<string>\",\n  \"candidate_retriever_ids\": [\n    \"<string>\"\n  ],\n  \"session_filter\": {\n    \"retriever_ids\": [\n      \"<string>\"\n    ],\n    \"taxonomy_node_ids\": [\n      \"<string>\"\n    ],\n    \"time_range\": {\n      \"start\": \"2023-11-07T05:31:56Z\",\n      \"end\": \"2023-11-07T05:31:56Z\"\n    },\n    \"min_interactions\": 1,\n    \"interaction_types\": [\n      \"<string>\"\n    ],\n    \"sample_strategy\": \"random\",\n    \"interaction_weights\": {\n      \"weights\": {}\n    }\n  },\n  \"session_count\": 1000\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.mixpeek.com/v1/retrievers/benchmarks")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"benchmark_name\": \"<string>\",\n  \"baseline_retriever_id\": \"<string>\",\n  \"candidate_retriever_ids\": [\n    \"<string>\"\n  ],\n  \"session_filter\": {\n    \"retriever_ids\": [\n      \"<string>\"\n    ],\n    \"taxonomy_node_ids\": [\n      \"<string>\"\n    ],\n    \"time_range\": {\n      \"start\": \"2023-11-07T05:31:56Z\",\n      \"end\": \"2023-11-07T05:31:56Z\"\n    },\n    \"min_interactions\": 1,\n    \"interaction_types\": [\n      \"<string>\"\n    ],\n    \"sample_strategy\": \"random\",\n    \"interaction_weights\": {\n      \"weights\": {}\n    }\n  },\n  \"session_count\": 1000\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.mixpeek.com/v1/retrievers/benchmarks")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"benchmark_name\": \"<string>\",\n  \"baseline_retriever_id\": \"<string>\",\n  \"candidate_retriever_ids\": [\n    \"<string>\"\n  ],\n  \"session_filter\": {\n    \"retriever_ids\": [\n      \"<string>\"\n    ],\n    \"taxonomy_node_ids\": [\n      \"<string>\"\n    ],\n    \"time_range\": {\n      \"start\": \"2023-11-07T05:31:56Z\",\n      \"end\": \"2023-11-07T05:31:56Z\"\n    },\n    \"min_interactions\": 1,\n    \"interaction_types\": [\n      \"<string>\"\n    ],\n    \"sample_strategy\": \"random\",\n    \"interaction_weights\": {\n      \"weights\": {}\n    }\n  },\n  \"session_count\": 1000\n}"

response = http.request(request)
puts response.read_body

{
  "benchmark_id": "<string>",
  "benchmark_name": "<string>",
  "baseline_retriever_id": "<string>",
  "candidate_retriever_ids": [
    "<string>"
  ],
  "session_count": 123,
  "created_at": "2023-11-07T05:31:56Z",
  "session_filter": {
    "retriever_ids": [
      "<string>"
    ],
    "taxonomy_node_ids": [
      "<string>"
    ],
    "time_range": {
      "start": "2023-11-07T05:31:56Z",
      "end": "2023-11-07T05:31:56Z"
    },
    "min_interactions": 1,
    "interaction_types": [
      "<string>"
    ],
    "sample_strategy": "random",
    "interaction_weights": {
      "weights": {}
    }
  },
  "results": [
    {
      "retriever_id": "<string>",
      "retriever_name": "<string>",
      "pipeline_hash": "<string>",
      "metrics": {
        "ndcg_at_k": {},
        "mean_rank_clicked": 123,
        "recall_at_k": {},
        "avg_position_delta": 123,
        "items_promoted": 1,
        "items_demoted": 1,
        "sessions_improved": 1,
        "sessions_degraded": 1,
        "sessions_neutral": 1,
        "mean_rank_purchased": 123
      },
      "latency": {
        "p50_ms": 1,
        "p90_ms": 1,
        "p99_ms": 1,
        "mean_ms": 1,
        "stage_latencies": {}
      },
      "failed_sessions": 1,
      "taxonomy_deltas": {},
      "error_summary": {}
    }
  ],
  "comparison": {
    "baseline_retriever_id": "<string>",
    "comparisons": [
      {
        "candidate_retriever_id": "<string>",
        "ndcg_delta": {},
        "recall_delta": {},
        "latency_delta_ms": 123,
        "p_value": 123,
        "confidence_interval": {
          "[0]": 123,
          "[1]": 123
        },
        "taxonomy_wins": [
          "<string>"
        ],
        "taxonomy_losses": [
          "<string>"
        ]
      }
    ],
    "recommendation": "<string>"
  },
  "started_at": "2023-11-07T05:31:56Z",
  "completed_at": "2023-11-07T05:31:56Z",
  "error_message": "<string>"
}

{
  "status": 123,
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>",
    "details": {}
  },
  "success": false
}

{
  "status": 123,
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>",
    "details": {}
  },
  "success": false
}

{
  "status": 123,
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>",
    "details": {}
  },
  "success": false
}

{
  "status": 123,
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>",
    "details": {}
  },
  "success": false
}

{
  "detail": [
    {
      "loc": [
        "<string>"
      ],
      "msg": "<string>",
      "type": "<string>",
      "input": "<unknown>",
      "ctx": {}
    }
  ]
}

{
  "status": 123,
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>",
    "details": {}
  },
  "success": false
}

POST

retrievers

benchmarks

Create benchmark

curl --request POST \
  --url https://api.mixpeek.com/v1/retrievers/benchmarks \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "benchmark_name": "<string>",
  "baseline_retriever_id": "<string>",
  "candidate_retriever_ids": [
    "<string>"
  ],
  "session_filter": {
    "retriever_ids": [
      "<string>"
    ],
    "taxonomy_node_ids": [
      "<string>"
    ],
    "time_range": {
      "start": "2023-11-07T05:31:56Z",
      "end": "2023-11-07T05:31:56Z"
    },
    "min_interactions": 1,
    "interaction_types": [
      "<string>"
    ],
    "sample_strategy": "random",
    "interaction_weights": {
      "weights": {}
    }
  },
  "session_count": 1000
}
'

import requests

url = "https://api.mixpeek.com/v1/retrievers/benchmarks"

payload = {
    "benchmark_name": "<string>",
    "baseline_retriever_id": "<string>",
    "candidate_retriever_ids": ["<string>"],
    "session_filter": {
        "retriever_ids": ["<string>"],
        "taxonomy_node_ids": ["<string>"],
        "time_range": {
            "start": "2023-11-07T05:31:56Z",
            "end": "2023-11-07T05:31:56Z"
        },
        "min_interactions": 1,
        "interaction_types": ["<string>"],
        "sample_strategy": "random",
        "interaction_weights": { "weights": {} }
    },
    "session_count": 1000
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    benchmark_name: '<string>',
    baseline_retriever_id: '<string>',
    candidate_retriever_ids: ['<string>'],
    session_filter: {
      retriever_ids: ['<string>'],
      taxonomy_node_ids: ['<string>'],
      time_range: {start: '2023-11-07T05:31:56Z', end: '2023-11-07T05:31:56Z'},
      min_interactions: 1,
      interaction_types: ['<string>'],
      sample_strategy: 'random',
      interaction_weights: {weights: {}}
    },
    session_count: 1000
  })
};

fetch('https://api.mixpeek.com/v1/retrievers/benchmarks', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.mixpeek.com/v1/retrievers/benchmarks",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'benchmark_name' => '<string>',
    'baseline_retriever_id' => '<string>',
    'candidate_retriever_ids' => [
        '<string>'
    ],
    'session_filter' => [
        'retriever_ids' => [
                '<string>'
        ],
        'taxonomy_node_ids' => [
                '<string>'
        ],
        'time_range' => [
                'start' => '2023-11-07T05:31:56Z',
                'end' => '2023-11-07T05:31:56Z'
        ],
        'min_interactions' => 1,
        'interaction_types' => [
                '<string>'
        ],
        'sample_strategy' => 'random',
        'interaction_weights' => [
                'weights' => [
                                
                ]
        ]
    ],
    'session_count' => 1000
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.mixpeek.com/v1/retrievers/benchmarks"

	payload := strings.NewReader("{\n  \"benchmark_name\": \"<string>\",\n  \"baseline_retriever_id\": \"<string>\",\n  \"candidate_retriever_ids\": [\n    \"<string>\"\n  ],\n  \"session_filter\": {\n    \"retriever_ids\": [\n      \"<string>\"\n    ],\n    \"taxonomy_node_ids\": [\n      \"<string>\"\n    ],\n    \"time_range\": {\n      \"start\": \"2023-11-07T05:31:56Z\",\n      \"end\": \"2023-11-07T05:31:56Z\"\n    },\n    \"min_interactions\": 1,\n    \"interaction_types\": [\n      \"<string>\"\n    ],\n    \"sample_strategy\": \"random\",\n    \"interaction_weights\": {\n      \"weights\": {}\n    }\n  },\n  \"session_count\": 1000\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.mixpeek.com/v1/retrievers/benchmarks")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"benchmark_name\": \"<string>\",\n  \"baseline_retriever_id\": \"<string>\",\n  \"candidate_retriever_ids\": [\n    \"<string>\"\n  ],\n  \"session_filter\": {\n    \"retriever_ids\": [\n      \"<string>\"\n    ],\n    \"taxonomy_node_ids\": [\n      \"<string>\"\n    ],\n    \"time_range\": {\n      \"start\": \"2023-11-07T05:31:56Z\",\n      \"end\": \"2023-11-07T05:31:56Z\"\n    },\n    \"min_interactions\": 1,\n    \"interaction_types\": [\n      \"<string>\"\n    ],\n    \"sample_strategy\": \"random\",\n    \"interaction_weights\": {\n      \"weights\": {}\n    }\n  },\n  \"session_count\": 1000\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.mixpeek.com/v1/retrievers/benchmarks")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"benchmark_name\": \"<string>\",\n  \"baseline_retriever_id\": \"<string>\",\n  \"candidate_retriever_ids\": [\n    \"<string>\"\n  ],\n  \"session_filter\": {\n    \"retriever_ids\": [\n      \"<string>\"\n    ],\n    \"taxonomy_node_ids\": [\n      \"<string>\"\n    ],\n    \"time_range\": {\n      \"start\": \"2023-11-07T05:31:56Z\",\n      \"end\": \"2023-11-07T05:31:56Z\"\n    },\n    \"min_interactions\": 1,\n    \"interaction_types\": [\n      \"<string>\"\n    ],\n    \"sample_strategy\": \"random\",\n    \"interaction_weights\": {\n      \"weights\": {}\n    }\n  },\n  \"session_count\": 1000\n}"

response = http.request(request)
puts response.read_body

{
  "benchmark_id": "<string>",
  "benchmark_name": "<string>",
  "baseline_retriever_id": "<string>",
  "candidate_retriever_ids": [
    "<string>"
  ],
  "session_count": 123,
  "created_at": "2023-11-07T05:31:56Z",
  "session_filter": {
    "retriever_ids": [
      "<string>"
    ],
    "taxonomy_node_ids": [
      "<string>"
    ],
    "time_range": {
      "start": "2023-11-07T05:31:56Z",
      "end": "2023-11-07T05:31:56Z"
    },
    "min_interactions": 1,
    "interaction_types": [
      "<string>"
    ],
    "sample_strategy": "random",
    "interaction_weights": {
      "weights": {}
    }
  },
  "results": [
    {
      "retriever_id": "<string>",
      "retriever_name": "<string>",
      "pipeline_hash": "<string>",
      "metrics": {
        "ndcg_at_k": {},
        "mean_rank_clicked": 123,
        "recall_at_k": {},
        "avg_position_delta": 123,
        "items_promoted": 1,
        "items_demoted": 1,
        "sessions_improved": 1,
        "sessions_degraded": 1,
        "sessions_neutral": 1,
        "mean_rank_purchased": 123
      },
      "latency": {
        "p50_ms": 1,
        "p90_ms": 1,
        "p99_ms": 1,
        "mean_ms": 1,
        "stage_latencies": {}
      },
      "failed_sessions": 1,
      "taxonomy_deltas": {},
      "error_summary": {}
    }
  ],
  "comparison": {
    "baseline_retriever_id": "<string>",
    "comparisons": [
      {
        "candidate_retriever_id": "<string>",
        "ndcg_delta": {},
        "recall_delta": {},
        "latency_delta_ms": 123,
        "p_value": 123,
        "confidence_interval": {
          "[0]": 123,
          "[1]": 123
        },
        "taxonomy_wins": [
          "<string>"
        ],
        "taxonomy_losses": [
          "<string>"
        ]
      }
    ],
    "recommendation": "<string>"
  },
  "started_at": "2023-11-07T05:31:56Z",
  "completed_at": "2023-11-07T05:31:56Z",
  "error_message": "<string>"
}

{
  "status": 123,
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>",
    "details": {}
  },
  "success": false
}

{
  "status": 123,
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>",
    "details": {}
  },
  "success": false
}

{
  "status": 123,
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>",
    "details": {}
  },
  "success": false
}

{
  "status": 123,
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>",
    "details": {}
  },
  "success": false
}

{
  "detail": [
    {
      "loc": [
        "<string>"
      ],
      "msg": "<string>",
      "type": "<string>",
      "input": "<unknown>",
      "ctx": {}
    }
  ]
}

{
  "status": 123,
  "error": {
    "message": "<string>",
    "type": "<string>",
    "code": "<string>",
    "details": {}
  },
  "success": false
}

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Body

application/json

Request to create a new benchmark run.

benchmark_name

string

required

Human-readable name for this benchmark.

Required string length: 1 - 255

baseline_retriever_id

string

required

ID of the baseline retriever pipeline to compare against.

candidate_retriever_ids

string[]

required

IDs of candidate retriever pipelines to evaluate.

Minimum array length: 1

session_filter

SessionFilter · object | null

Optional filter criteria for selecting sessions to replay.

Show child attributes

session_count

integer

default:1000

Number of sessions to include in the benchmark.

Required range: 10 <= x <= 10000

Response

Successful Response

Response containing benchmark details and results.

benchmark_id

string

required

Unique benchmark identifier.

benchmark_name

string

required

Human-readable name.

baseline_retriever_id

string

required

Baseline retriever ID.

candidate_retriever_ids

string[]

required

Candidate retriever IDs.

session_count

integer

required

Number of sessions in benchmark.

status

enum<string>

required

Current benchmark status.

Available options:

pending,

building_sessions,

replaying,

computing_metrics,

completed,

failed

created_at

string<date-time>

required

Creation timestamp.

session_filter

SessionFilter · object | null

Filter criteria used.

Show child attributes

results

BenchmarkResult · object[] | null

Results per pipeline (available when completed).

Show child attributes

comparison

BenchmarkComparison · object | null

Statistical comparison (available when completed).

Show child attributes

started_at

string<date-time> | null

Execution start time.

completed_at

string<date-time> | null

Completion time.

error_message

string | null

Error message if failed.

List benchmarks Get benchmark

Organization

Namespaces

Buckets

Feature Extractors

Batch Queue

Collections

Documents

Retrievers

Taxonomies

Clusters

Triggers

Alerts

Webhooks

Apps

Agent Sessions

Annotations

Templates

Manifest

Discovery

Analytics

Notifications

Tasks

Inference

Resource Search

Pricing

Create benchmark

Authorizations

Body

Response