Replay evaluations onto a new test case revision

Re-runs the evaluations that any revision in testCaseGroupId was scored with against the new test case revision newTestCaseId, scoped to the given products. Because an evaluation is bound to a test case only through its session, fresh inference is generated for the new revision on each version that previously ran the family (production sessions excluded), then evaluated with the metrics that ran there. De-duplication skips a (version, metric) pair once it has a non-deleted evaluation against newTestCaseId, and skips a whole version once any non-deleted session exists for newTestCaseId — the session is created before this call returns, so a repeated or retried call re-dispatches nothing. This is still not a transactional guarantee for truly simultaneous calls. newTestCaseId must be the active head of its family.

POST /evaluations/fromTestCases

Replay evaluations onto a new test case revision

cURL

curl --request POST \
  --url https://api.galtea.ai/evaluations/fromTestCases \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "newTestCaseId": "testCase_123",
  "testCaseGroupId": "group_123",
  "productIds": [
    "prod_1",
    "prod_2"
  ]
}
'

import requests

url = "https://api.galtea.ai/evaluations/fromTestCases"

payload = {
    "newTestCaseId": "testCase_123",
    "testCaseGroupId": "group_123",
    "productIds": ["prod_1", "prod_2"]
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    newTestCaseId: 'testCase_123',
    testCaseGroupId: 'group_123',
    productIds: ['prod_1', 'prod_2']
  })
};

fetch('https://api.galtea.ai/evaluations/fromTestCases', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.galtea.ai/evaluations/fromTestCases",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'newTestCaseId' => 'testCase_123',
    'testCaseGroupId' => 'group_123',
    'productIds' => [
        'prod_1',
        'prod_2'
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.galtea.ai/evaluations/fromTestCases"

	payload := strings.NewReader("{\n  \"newTestCaseId\": \"testCase_123\",\n  \"testCaseGroupId\": \"group_123\",\n  \"productIds\": [\n    \"prod_1\",\n    \"prod_2\"\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.galtea.ai/evaluations/fromTestCases")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"newTestCaseId\": \"testCase_123\",\n  \"testCaseGroupId\": \"group_123\",\n  \"productIds\": [\n    \"prod_1\",\n    \"prod_2\"\n  ]\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.galtea.ai/evaluations/fromTestCases")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"newTestCaseId\": \"testCase_123\",\n  \"testCaseGroupId\": \"group_123\",\n  \"productIds\": [\n    \"prod_1\",\n    \"prod_2\"\n  ]\n}"

response = http.request(request)
puts response.read_body

202

{
  "scheduled": 123,
  "skipped": 123,
  "errors": [
    "<string>"
  ]
}

{
  "error": "Error type",
  "message": "Error message description"
}

{
  "error": "Error type",
  "message": "Error message description"
}

{
  "error": "Error type",
  "message": "Error message description"
}

Authorizations

Authorization · string · header · required

API key authorization. Pass your API key in the Authorization header as a Bearer token. Both new (gsk_*) and legacy (gsk-) API keys are accepted, e.g. Authorization: Bearer gsk_... or Authorization: Bearer gsk-....

Body

application/json

newTestCaseId · string · required

The new test case revision to replay onto. Must be the active head of its family.

Minimum string length: 1

Example:

"testCase_123"

testCaseGroupId · string · required

The revision family whose past evaluations are replayed. Must match the test case.

Minimum string length: 1

Example:

"group_123"

productIds · string[] · required

Products whose past evaluations are replayed.

Minimum array length: 1

Minimum string length: 1

Example:

["prod_1", "prod_2"]

Response

202

application/json

Replay job accepted

scheduled · integer · required

Count of (version, metric) pairs dispatched for fresh inference

skipped · integer · required

Count of (version, metric) pairs skipped (dispatch failures)

errors · string[] · required

Per-version dispatch error messages