{"schema_version":"alhena-research-lab/tool-summaries-v2","generated_at":"2026-09-23T04:06:41.748Z","publisher":{"@type":"Organization","@id":"https://evals.alhena.ai/#organization","name":"Alhena Research Lab","url":"https://evals.alhena.ai/","logo":"https://evals.alhena.ai/brand/alhena-logo.svg","parentOrganization":{"@type":"Organization","name":"Alhena","url":"https://alhena.ai/"},"sameAs":["https://alhena.ai/","https://github.com/mbahacker/comparison-lab"]},"scope":"Latest compatible approved policy-resolution study per tool, selected by capture end date then publication date. Policy-compliant resolution, answer quality, full-answer speed and composite are separate measures. Public-session scope does not establish completed backend actions or a universal vendor ranking.","methodology_url":"https://evals.alhena.ai/studies/policy-resolution-v1","quality_pilot_archive_url":"https://evals.alhena.ai/quality-pilot-scores.json","evidence_access":"Public summaries only. Detailed evidence requires a verified work email.","tools":[{"id":"alhena-ai-e8955b5e31","name":"Alhena","website":"https://alhena.ai/","studySlug":"alhena-gorgias-policy-resolution-2026-09-21","studyTitle":"Alhena and Gorgias: policy-compliant resolution","protocol":"policy-resolution-v1","publishedAt":"2026-09-22T04:07:09.170Z","captureStartAt":"2026-09-21T14:13:43.224Z","captureEndAt":"2026-09-22T00:30:54.972Z","registeredStores":5,"shopping":{"policyResolution":{"value":64.8,"explanation":"Verified correct answer or policy-prescribed next step, averaged equally by conversation and storefront. This does not measure downstream case completion."},"quality":{"value":83,"explanation":"The original fixed answer-quality rubric. Unchanged captures retain their original judgments; repaired captures receive a fresh judgment and audit."},"speed":{"value":53.7,"explanation":"Full-answer completion 11.8 seconds; 100 at 3 seconds, zero at 22 seconds, bounded between 0 and 100."},"composite":{"value":68.4,"explanation":"40% policy-compliant resolution, 35% quality, 25% speed."},"coverage":{"plannedCheckpoints":250,"attemptedCheckpoints":242,"observedCheckpoints":242,"submittedCheckpoints":242,"assessedCheckpoints":242,"unassessableCheckpoints":0,"attainedCheckpoints":158,"policyUnverifiedCheckpoints":64,"includedContexts":25,"excludedContexts":0,"includedStores":5,"excludedStores":0,"qualityEligibleContexts":25,"qualityEligibleStores":5,"originalCaptures":23,"repairedCaptures":2}},"support":{"policyResolution":{"value":72.8,"explanation":"Verified correct answer or policy-prescribed next step, averaged equally by conversation and storefront. This does not measure downstream case completion."},"quality":{"value":95,"explanation":"The original fixed answer-quality rubric. Unchanged captures retain their original judgments; repaired captures receive a fresh judgment and audit."},"speed":{"value":70,"explanation":"Full-answer completion 8.7 seconds; 100 at 3 seconds, zero at 22 seconds, bounded between 0 and 100."},"composite":{"value":81.4,"explanation":"50% policy-compliant resolution, 40% quality, 10% speed."},"coverage":{"plannedCheckpoints":250,"attemptedCheckpoints":250,"observedCheckpoints":250,"submittedCheckpoints":250,"assessedCheckpoints":250,"unassessableCheckpoints":0,"attainedCheckpoints":182,"policyUnverifiedCheckpoints":63,"includedContexts":25,"excludedContexts":0,"includedStores":5,"excludedStores":0,"qualityEligibleContexts":25,"qualityEligibleStores":5,"originalCaptures":19,"repairedCaptures":6}},"overallComposite":{"value":74.9,"explanation":"Equal mean of the two unrounded lane composites; only available when both lanes meet coverage floors."},"url":"https://evals.alhena.ai/tools/alhena-ai-e8955b5e31","source_study_url":"https://evals.alhena.ai/studies/alhena-gorgias-policy-resolution-2026-09-21"},{"id":"gorgias-com-b56a925583","name":"Gorgias","website":"https://www.gorgias.com/","studySlug":"alhena-gorgias-policy-resolution-2026-09-21","studyTitle":"Alhena and Gorgias: policy-compliant resolution","protocol":"policy-resolution-v1","publishedAt":"2026-09-22T04:07:09.170Z","captureStartAt":"2026-09-21T14:13:43.224Z","captureEndAt":"2026-09-22T00:30:54.972Z","registeredStores":5,"shopping":{"policyResolution":{"value":60.8,"explanation":"Verified correct answer or policy-prescribed next step, averaged equally by conversation and storefront. This does not measure downstream case completion."},"quality":{"value":72,"explanation":"The original fixed answer-quality rubric. Unchanged captures retain their original judgments; repaired captures receive a fresh judgment and audit."},"speed":{"value":25.3,"explanation":"Full-answer completion 17.2 seconds; 100 at 3 seconds, zero at 22 seconds, bounded between 0 and 100."},"composite":{"value":55.8,"explanation":"40% policy-compliant resolution, 35% quality, 25% speed."},"coverage":{"plannedCheckpoints":250,"attemptedCheckpoints":187,"observedCheckpoints":187,"submittedCheckpoints":187,"assessedCheckpoints":164,"unassessableCheckpoints":23,"attainedCheckpoints":109,"policyUnverifiedCheckpoints":37,"includedContexts":20,"excludedContexts":5,"includedStores":4,"excludedStores":1,"qualityEligibleContexts":17,"qualityEligibleStores":4,"originalCaptures":23,"repairedCaptures":2}},"support":{"policyResolution":{"value":68,"explanation":"Verified correct answer or policy-prescribed next step, averaged equally by conversation and storefront. This does not measure downstream case completion."},"quality":{"value":94,"explanation":"The original fixed answer-quality rubric. Unchanged captures retain their original judgments; repaired captures receive a fresh judgment and audit."},"speed":{"value":41.6,"explanation":"Full-answer completion 14.1 seconds; 100 at 3 seconds, zero at 22 seconds, bounded between 0 and 100."},"composite":{"value":75.8,"explanation":"50% policy-compliant resolution, 40% quality, 10% speed."},"coverage":{"plannedCheckpoints":250,"attemptedCheckpoints":205,"observedCheckpoints":205,"submittedCheckpoints":205,"assessedCheckpoints":180,"unassessableCheckpoints":25,"attainedCheckpoints":122,"policyUnverifiedCheckpoints":45,"includedContexts":18,"excludedContexts":7,"includedStores":4,"excludedStores":1,"qualityEligibleContexts":18,"qualityEligibleStores":4,"originalCaptures":23,"repairedCaptures":2}},"overallComposite":{"value":65.8,"explanation":"Equal mean of the two unrounded lane composites; only available when both lanes meet coverage floors."},"url":"https://evals.alhena.ai/tools/gorgias-com-b56a925583","source_study_url":"https://evals.alhena.ai/studies/alhena-gorgias-policy-resolution-2026-09-21"},{"id":"sierra-ai-98c6f6e402","name":"Sierra","website":"https://sierra.ai/","studySlug":"sierra-ai-98c6f6e402-9cba991e","studyTitle":"Sierra: policy-resolution study","protocol":"policy-resolution-v1","publishedAt":"2026-09-23T00:59:21.323Z","captureStartAt":"2026-09-22T15:03:08.664Z","captureEndAt":"2026-09-23T00:53:06.466Z","registeredStores":5,"shopping":{"policyResolution":{"value":61.1,"explanation":"Verified correct answer or policy-prescribed next step, averaged equally by conversation and storefront. This does not measure downstream case completion."},"quality":{"value":48,"explanation":"The original fixed answer-quality rubric. Unchanged captures retain their original judgments; repaired captures receive a fresh judgment and audit."},"speed":{"value":71.1,"explanation":"Full-answer completion 8.5 seconds; 100 at 3 seconds, zero at 22 seconds, bounded between 0 and 100."},"composite":{"value":59,"explanation":"40% policy-compliant resolution, 35% quality, 25% speed."},"coverage":{"plannedCheckpoints":250,"attemptedCheckpoints":183,"observedCheckpoints":183,"submittedCheckpoints":183,"assessedCheckpoints":183,"unassessableCheckpoints":0,"attainedCheckpoints":104,"policyUnverifiedCheckpoints":38,"includedContexts":25,"excludedContexts":0,"includedStores":5,"excludedStores":0,"qualityEligibleContexts":20,"qualityEligibleStores":4,"originalCaptures":24,"repairedCaptures":1}},"support":{"policyResolution":{"value":40,"explanation":"Verified correct answer or policy-prescribed next step, averaged equally by conversation and storefront. This does not measure downstream case completion."},"quality":{"value":86,"explanation":"The original fixed answer-quality rubric. Unchanged captures retain their original judgments; repaired captures receive a fresh judgment and audit."},"speed":{"value":71.6,"explanation":"Full-answer completion 8.4 seconds; 100 at 3 seconds, zero at 22 seconds, bounded between 0 and 100."},"composite":{"value":61.6,"explanation":"50% policy-compliant resolution, 40% quality, 10% speed."},"coverage":{"plannedCheckpoints":250,"attemptedCheckpoints":242,"observedCheckpoints":242,"submittedCheckpoints":242,"assessedCheckpoints":242,"unassessableCheckpoints":0,"attainedCheckpoints":96,"policyUnverifiedCheckpoints":137,"includedContexts":25,"excludedContexts":0,"includedStores":5,"excludedStores":0,"qualityEligibleContexts":24,"qualityEligibleStores":5,"originalCaptures":25,"repairedCaptures":0}},"overallComposite":{"value":60.3,"explanation":"Equal mean of the two unrounded lane composites; only available when both lanes meet coverage floors."},"url":"https://evals.alhena.ai/tools/sierra-ai-98c6f6e402","source_study_url":"https://evals.alhena.ai/studies/sierra-ai-98c6f6e402-9cba991e"}]}