> ## Documentation Index
> Fetch the complete documentation index at: https://agent-observability-docs.splunk.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Conversation Quality

> Learn how to measure the quality of a conversation that a user has with a chatbot

export const SupportedModalities = ({modalities}) => {
  const iconProps = {
    fill: "none",
    height: 20,
    width: 20,
    stroke: "currentColor",
    strokeLinecap: "round",
    strokeLinejoin: "round",
    strokeWidth: 2,
    viewBox: "0 0 24 24",
    xmlns: "http://www.w3.org/2000/svg"
  };
  const IconText = () => <svg {...iconProps} aria-hidden="true">
      <circle cx="12" cy="12" r="9" />
      <path d="M8 8h8M12 8v8" />
    </svg>;
  const IconImage = () => <svg {...iconProps} aria-hidden="true">
      <rect height="18" rx="2" width="18" x="3" y="3" />
      <circle cx="8.5" cy="8.5" r="1.5" />
      <path d="m21 15-5-5L5 21" />
    </svg>;
  const IconMusic = () => <svg {...iconProps} aria-hidden="true">
      <path d="M9 18V5l10-2v13" />
      <circle cx="6" cy="18" r="3" />
      <circle cx="16" cy="16" r="3" />
    </svg>;
  const modalityIcons = {
    Text: IconText,
    "Image/PDF": IconImage,
    Audio: IconMusic
  };
  return <div style={{
    alignItems: "center",
    display: "flex",
    gap: "0.5rem",
    marginBottom: "1.25rem",
    marginTop: "0.75rem"
  }}>
      <span style={{
    fontSize: "0.875rem",
    fontWeight: 500
  }}>Supported modalities:</span>
      <span aria-label={`Supported modalities: ${modalities.join(", ")}`} role="group" style={{
    display: "inline-flex",
    alignItems: "center",
    gap: "0.5rem"
  }}>
        {modalities.map(modality => {
    const Icon = modalityIcons[modality];
    if (!Icon) return null;
    return <span aria-label={modality} key={modality} role="img" style={{
      display: "inline-flex",
      flexShrink: 0
    }} title={modality}>
              <Icon />
            </span>;
  })}
      </span>
    </div>;
};

export const BooleanClassificationReport = ({report, negativeLabel = "Not Advanced", positiveLabel = "Advanced", negativeClass = "False", positiveClass = "True", maxWidth = 520}) => {
  const parseReport = reportStr => {
    const lines = reportStr.trim().split('\n').filter(line => line.trim());
    const result = {
      classes: [],
      accuracy: null,
      macroAvg: null,
      weightedAvg: null,
      totalSupport: null
    };
    for (const line of lines) {
      const parts = line.trim().split(/\s+/);
      if (parts[0] === 'precision') continue;
      if (parts.length >= 5 && !['accuracy', 'macro', 'weighted'].includes(parts[0])) {
        result.classes.push({
          name: parts[0],
          precision: parseFloat(parts[1]),
          recall: parseFloat(parts[2]),
          f1: parseFloat(parts[3]),
          support: parseInt(parts[4], 10)
        });
      }
      if (parts[0] === 'accuracy') {
        result.accuracy = parseFloat(parts[1]);
        result.totalSupport = parseInt(parts[2], 10);
      }
      if (parts[0] === 'macro' && parts[1] === 'avg') {
        result.macroAvg = {
          precision: parseFloat(parts[2]),
          recall: parseFloat(parts[3]),
          f1: parseFloat(parts[4]),
          support: parseInt(parts[5], 10)
        };
      }
      if (parts[0] === 'weighted' && parts[1] === 'avg') {
        result.weightedAvg = {
          precision: parseFloat(parts[2]),
          recall: parseFloat(parts[3]),
          f1: parseFloat(parts[4]),
          support: parseInt(parts[5], 10)
        };
      }
    }
    return result;
  };
  const parsed = parseReport(report);
  if (parsed.classes.length < 2) {
    return <div style={{
      color: "red",
      padding: "1rem",
      border: "1px solid red"
    }}>BooleanClassificationReport: Could not parse report. Expected at least 2 classes.</div>;
  }
  const negClass = parsed.classes.find(c => c.name === negativeClass) || parsed.classes[0];
  const posClass = parsed.classes.find(c => c.name === positiveClass) || parsed.classes[1];
  const tnPlusFp = negClass.support;
  const tpPlusFn = posClass.support;
  const tn = Math.round(negClass.recall * tnPlusFp);
  const fp = tnPlusFp - tn;
  const tp = Math.round(posClass.recall * tpPlusFn);
  const fn = tpPlusFn - tp;
  const tnPct = tn / tnPlusFp * 100;
  const fpPct = fp / tnPlusFp * 100;
  const fnPct = fn / tpPlusFn * 100;
  const tpPct = tp / tpPlusFn * 100;
  const rowStyle = {
    borderBottom: "1px solid rgba(148, 163, 184, 0.3)"
  };
  const cellStyle = {
    padding: "0.5rem 0.125rem"
  };
  const centerCellStyle = {
    textAlign: "center",
    padding: "0.5rem 0.125rem"
  };
  return <div>
      {}
      <table style={{
    width: "auto",
    borderCollapse: "collapse",
    marginBottom: "1.5rem",
    fontSize: "0.875rem"
  }}>
        <thead>
          <tr style={{
    borderBottom: "2px solid rgba(148, 163, 184, 0.5)"
  }}>
            <th style={{
    textAlign: "center",
    padding: "0.5rem 0.125rem",
    fontWeight: "600"
  }}></th>
            <th style={{
    textAlign: "center",
    padding: "0.5rem 0.125rem",
    fontWeight: "600"
  }}>Precision</th>
            <th style={{
    textAlign: "center",
    padding: "0.5rem 0.125rem",
    fontWeight: "600"
  }}>Recall</th>
            <th style={{
    textAlign: "center",
    padding: "0.5rem 0.125rem",
    fontWeight: "600"
  }}>F1-Score</th>
          </tr>
        </thead>
        <tbody>
          {}
          <tr style={rowStyle}>
            <td style={cellStyle}>{negativeLabel}</td>
            <td style={centerCellStyle}>{negClass.precision.toFixed(2)}</td>
            <td style={centerCellStyle}>{negClass.recall.toFixed(2)}</td>
            <td style={centerCellStyle}>{negClass.f1.toFixed(2)}</td>
          </tr>
          <tr style={rowStyle}>
            <td style={cellStyle}>{positiveLabel}</td>
            <td style={centerCellStyle}>{posClass.precision.toFixed(2)}</td>
            <td style={centerCellStyle}>{posClass.recall.toFixed(2)}</td>
            <td style={centerCellStyle}>{posClass.f1.toFixed(2)}</td>
          </tr>
          
        </tbody>
      </table>

      {}
      <BooleanConfusionMatrix actualNegativeLabel={negativeLabel} actualPositiveLabel={positiveLabel} predictedNegativeLabel={negativeLabel} predictedPositiveLabel={positiveLabel} tnPct={tnPct.toString()} fpPct={fpPct.toString()} fnPct={fnPct.toString()} tpPct={tpPct.toString()} displayFormat="fraction" maxWidth={maxWidth} />
    </div>;
};

export const BooleanConfusionMatrix = ({actualNegativeLabel = "Not Advanced", actualPositiveLabel = "Advanced", predictedNegativeLabel = "Not Advanced", predictedPositiveLabel = "Advanced", tnCount, tnPct, fpCount, fpPct, fnCount, fnPct, tpCount, tpPct, matrix, maxWidth = 520, displayFormat = "percentage", fractionDigits = 3, percentageDigits = 1, titlePrefix = ""}) => {
  const parseNum = val => val !== undefined && val !== null ? Number(val) : undefined;
  const clampPct = pct => Math.max(0, Math.min(100, Number(pct) || 0));
  const formatValue = pct => {
    const p = clampPct(pct);
    if (displayFormat === "fraction") {
      const digits = Number.isFinite(Number(fractionDigits)) ? Number(fractionDigits) : 3;
      return (p / 100).toFixed(digits);
    }
    const digits = Number.isFinite(Number(percentageDigits)) ? Number(percentageDigits) : 1;
    return `${p.toFixed(digits)}%`;
  };
  const palette = ["#f8fafc", "#eff6ff", "#dbeafe", "#bfdbfe", "#93c5fd", "#60a5fa", "#3b82f6", "#2563eb", "#1d4ed8", "#1e40af"];
  const getBg = pct => {
    const p = clampPct(pct);
    const idx = p === 100 ? 9 : Math.floor(p / 10);
    return palette[idx];
  };
  const getColor = pct => clampPct(pct) >= 60 ? "#ffffff" : "#1e3a8a";
  const rawTn = parseNum(tnCount);
  const rawFp = parseNum(fpCount);
  const rawFn = parseNum(fnCount);
  const rawTp = parseNum(tpCount);
  const rawTnPct = parseNum(tnPct);
  const rawFpPct = parseNum(fpPct);
  const rawFnPct = parseNum(fnPct);
  const rawTpPct = parseNum(tpPct);
  const hasCounts = rawTn !== undefined && rawFp !== undefined && rawFn !== undefined && rawTp !== undefined;
  const hasPcts = rawTnPct !== undefined && rawFpPct !== undefined && rawFnPct !== undefined && rawTpPct !== undefined;
  let resolvedMatrix;
  let showCounts;
  if (matrix) {
    resolvedMatrix = matrix;
    showCounts = matrix.tn?.count !== undefined;
  } else if (hasCounts) {
    const actualNegTotal = rawTn + rawFp;
    const actualPosTotal = rawFn + rawTp;
    resolvedMatrix = {
      tn: {
        count: rawTn,
        pct: actualNegTotal > 0 ? rawTn / actualNegTotal * 100 : 0
      },
      fp: {
        count: rawFp,
        pct: actualNegTotal > 0 ? rawFp / actualNegTotal * 100 : 0
      },
      fn: {
        count: rawFn,
        pct: actualPosTotal > 0 ? rawFn / actualPosTotal * 100 : 0
      },
      tp: {
        count: rawTp,
        pct: actualPosTotal > 0 ? rawTp / actualPosTotal * 100 : 0
      }
    };
    showCounts = true;
  } else if (hasPcts) {
    resolvedMatrix = {
      tn: {
        pct: rawTnPct
      },
      fp: {
        pct: rawFpPct
      },
      fn: {
        pct: rawFnPct
      },
      tp: {
        pct: rawTpPct
      }
    };
    showCounts = false;
  } else {
    return <div style={{
      color: "red",
      padding: "1rem",
      border: "1px solid red"
    }}>BooleanConfusionMatrix: Provide either all counts or all percentages</div>;
  }
  const cellStyle = pct => ({
    background: getBg(pct),
    color: getColor(pct),
    padding: "1rem",
    textAlign: "center",
    borderRadius: "8px",
    aspectRatio: "1 / 1",
    width: "100%",
    display: "flex",
    flexDirection: "column",
    alignItems: "center",
    justifyContent: "center",
    border: "1px solid rgba(148, 163, 184, 0.35)"
  });
  const displayPredictedLabels = {
    left: predictedPositiveLabel,
    right: predictedNegativeLabel
  };
  const displayActualLabels = {
    top: actualPositiveLabel,
    bottom: actualNegativeLabel
  };
  const displayMatrix = {
    tl: resolvedMatrix.tp,
    tr: resolvedMatrix.fn,
    bl: resolvedMatrix.fp,
    br: resolvedMatrix.tn
  };
  return <div style={{
    maxWidth: maxWidth + "px",
    margin: "1rem 0"
  }}>
      <div style={{
    display: "grid",
    gridTemplateColumns: "auto auto 1fr 1fr",
    gridTemplateRows: "auto auto auto 1fr 1fr auto",
    gap: "2px"
  }}>
        {}
        <div></div>
        <div></div>
        <div style={{
    gridColumn: "3 / 5",
    textAlign: "center",
    padding: "0.5rem",
    fontWeight: "600",
    fontSize: "1rem"
  }}>
          {titlePrefix}Confusion Matrix (Normalized)
        </div>

        {}
        <div></div>
        <div></div>
        <div style={{
    gridColumn: "3 / 5",
    textAlign: "center",
    padding: "0.5rem",
    fontWeight: "600",
    fontSize: "0.875rem"
  }}>
          Predicted
        </div>

        {}
        <div></div>
        <div></div>
        <div style={{
    textAlign: "center",
    padding: "0.5rem",
    fontSize: "0.75rem",
    fontWeight: "500",
    display: "flex",
    alignItems: "center",
    justifyContent: "center"
  }}>{displayPredictedLabels.left}</div>
        <div style={{
    textAlign: "center",
    padding: "0.5rem",
    fontSize: "0.75rem",
    fontWeight: "500",
    display: "flex",
    alignItems: "center",
    justifyContent: "center"
  }}>{displayPredictedLabels.right}</div>

        {}
        <div style={{
    gridRow: "4 / 6",
    writingMode: "vertical-rl",
    transform: "rotate(180deg)",
    textAlign: "center",
    fontWeight: "600",
    fontSize: "0.875rem",
    padding: "0 0.5rem",
    display: "flex",
    alignItems: "center",
    justifyContent: "center"
  }}>
          Actual
        </div>
        <div style={{
    padding: "0.5rem",
    fontSize: "0.75rem",
    fontWeight: "500",
    display: "flex",
    alignItems: "center",
    justifyContent: "flex-end"
  }}>{displayActualLabels.top}</div>
        <div style={cellStyle(displayMatrix.tl.pct)}>
          {showCounts && <div style={{
    fontSize: "1.5rem",
    fontWeight: "700",
    lineHeight: 1
  }}>{displayMatrix.tl.count}</div>}
          <div style={{
    fontSize: showCounts ? "0.75rem" : "1rem",
    fontWeight: showCounts ? "400" : "700",
    opacity: showCounts ? 0.8 : 1
  }}>{formatValue(displayMatrix.tl.pct)}</div>
        </div>
        <div style={cellStyle(displayMatrix.tr.pct)}>
          {showCounts && <div style={{
    fontSize: "1.5rem",
    fontWeight: "700",
    lineHeight: 1
  }}>{displayMatrix.tr.count}</div>}
          <div style={{
    fontSize: showCounts ? "0.75rem" : "1rem",
    fontWeight: showCounts ? "400" : "700",
    opacity: showCounts ? 0.8 : 1
  }}>{formatValue(displayMatrix.tr.pct)}</div>
        </div>

        {}
        <div style={{
    padding: "0.5rem",
    fontSize: "0.75rem",
    fontWeight: "500",
    display: "flex",
    alignItems: "center",
    justifyContent: "flex-end"
  }}>{displayActualLabels.bottom}</div>
        <div style={cellStyle(displayMatrix.bl.pct)}>
          {showCounts && <div style={{
    fontSize: "1.5rem",
    fontWeight: "700",
    lineHeight: 1
  }}>{displayMatrix.bl.count}</div>}
          <div style={{
    fontSize: showCounts ? "0.75rem" : "1rem",
    fontWeight: showCounts ? "400" : "700",
    opacity: showCounts ? 0.8 : 1
  }}>{formatValue(displayMatrix.bl.pct)}</div>
        </div>
        <div style={cellStyle(displayMatrix.br.pct)}>
          {showCounts && <div style={{
    fontSize: "1.5rem",
    fontWeight: "700",
    lineHeight: 1
  }}>{displayMatrix.br.count}</div>}
          <div style={{
    fontSize: showCounts ? "0.75rem" : "1rem",
    fontWeight: showCounts ? "400" : "700",
    opacity: showCounts ? 0.8 : 1
  }}>{formatValue(displayMatrix.br.pct)}</div>
        </div>

        {}
        <div></div>
        <div></div>
        <div style={{
    gridColumn: "3 / 5",
    marginTop: "0.5rem",
    display: "flex",
    alignItems: "center",
    gap: "0.5rem"
  }}>
          <span style={{
    fontSize: "0.75rem",
    fontWeight: "500"
  }}>{displayFormat === "fraction" ? "0.0" : "0%"}</span>
          <div style={{
    display: "flex",
    flex: 1,
    height: "12px",
    borderRadius: "4px",
    overflow: "hidden",
    border: "1px solid rgba(148, 163, 184, 0.35)"
  }}>
            {palette.map((color, idx) => <div key={idx} style={{
    flex: 1,
    height: "100%",
    background: color
  }} />)}
          </div>
          <span style={{
    fontSize: "0.75rem",
    fontWeight: "500"
  }}>{displayFormat === "fraction" ? "1.0" : "100%"}</span>
        </div>
      </div>
    </div>;
};

export const EvaluatorWhenToUse = ({description, useCases}) => {
  return <Card>
      <div style={{
    display: 'flex',
    alignItems: 'center',
    gap: '0.5rem',
    marginBottom: '0.75rem'
  }}>
        <div style={{
    fontSize: '1.25rem',
    color: 'var(--primary-color)'
  }}>
          <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
            <path d="M12 22c5.523 0 10-4.477 10-10S17.523 2 12 2 2 6.477 2 12s4.477 10 10 10z" />
            <path d="m9 12 2 2 4-4" />
          </svg>
        </div>
        <h3 style={{
    margin: 0,
    fontSize: '1.25rem',
    fontWeight: '600'
  }}>When to Use This Evaluator</h3>
      </div>

      {description}

      {useCases != null && useCases.map((useCase, index) => <div key={index} style={{
    marginTop: "1rem",
    paddingTop: "0.75rem",
    borderTop: "1px solid rgba(209, 213, 219, 0.33)"
  }}>
          <strong>{useCase.title}</strong>{useCase.description ? `: ${useCase.description}` : ''}
        </div>)}
    </Card>;
};

export const DefinitionCard = ({children}) => {
  return <Card variant="secondary">
    <div style={{
    padding: '0.5rem',
    border: '5px solid var(--primary-light)',
    borderRadius: '0.5rem',
    fontSize: '1.3rem',
    lineHeight: '1.4',
    boxShadow: '0 0 10px 10px var(--primary-light)'
  }}>
        {children}
      </div>

</Card>;
};

<SupportedModalities modalities={["Text"]} />

<DefinitionCard>
  <strong>Conversation Quality</strong> is a binary evaluator that assesses whether a chatbot interaction left the user feeling satisfied and positive or frustrated and dissatisfied, based on tone, engagement, and overall experience.
</DefinitionCard>

The Conversation Quality evaluator measures user satisfaction across an entire chatbot session by analyzing tone, engagement, and sentiment. It classifies each conversation as GOOD or BAD depending on whether the user’s overall experience reflects positive engagement or frustration directed at the bot.

The evaluator focuses on conversational flow rather than task success, emphasizing how naturally and politely the user and bot interact. It excludes non-textual or purely action-based agent outputs (e.g., button clicks).

This is a **boolean** evaluator that indicates whether the conversation quality is good.

## Conversation Quality at a glance

| Property                   | Description          |
| :------------------------- | :------------------- |
| **Name**                   | Conversation Quality |
| **Category**               | Agentic AI           |
| **Can be applied to**      | Session              |
| **LLM-as-a-judge Support** | ✅                    |
| **Luna Support**           | ❌                    |
| **Value Type**             | Boolean              |

## When to use this evaluator

<EvaluatorWhenToUse
  description="Conversation Quality is a useful evaluator when working with chat bots or other tools with lots of user interaction"
  useCases={[
{
  title: "Chat tools where sentiment is critical",
  description: "Evaluating chatbot performance where capturing user sentiment is critical, such as customer support or counseling applications."
},
{
  title: "Customer satisfaction",
  description: "Monitoring and improving overall user satisfaction in conversational AI systems."
},
{
  title: "User experience quality",
  description: "Comparing different models or system versions based on experience quality rather than task completion."
}
]}
/>

## How to improve Conversation Quality scores

Some techniques to improve Conversation Quality scores are:

* Ensure bots provide clear, empathetic, and concise responses
* Detect and mitigate repeated clarification loops
* Train models to de-escalate external frustration effectively
* Log complete sessions to allow accurate tone assessment

Common issues that can cause low scores are:

* Mislabeling external frustration as bot-directed
* Incomplete logs
* Abrupt session truncation

## Performance Benchmarks

We evaluated Conversation Quality against human expert labels on an internal dataset of agentic conversation samples using top frontier models.

| Model                   | F1 (True) |
| :---------------------- | :-------: |
| GPT-4.1                 |    0.89   |
| GPT-4.1-mini (judges=3) |    0.85   |
| Claude Sonnet 4.5       |    0.85   |
| Gemini 3 Flash          |    0.88   |

### GPT-4.1 Classification Report

<BooleanClassificationReport
  report={`            precision    recall  f1-score   support

False       0.9100    0.8273    0.8667       110
True       0.8538    0.9250    0.8880       120

accuracy                          0.8783       230
macro avg     0.8819    0.8761    0.8773       230
weighted avg  0.8807    0.8783    0.8778       230`}
  negativeLabel="False"
  positiveLabel="True"
  negativeClass="False"
  positiveClass="True"
/>

<Note>
  Benchmarks based on internal evaluation dataset. Performance may vary by use case.
</Note>

## Related Resources

If you would like to dive deeper or start implementing Conversation Quality, check out the following resources:

### Examples

* Conversation Quality Examples - Log into Splunk Agent Observability and explore the "Conversation Quality" Agent Stream in the "Preset Evaluator Examples" Project to see this evaluator in action.

### How-to guides

* [Agentic AI Basic Example](/how-to-guides/agentic-ai/basic-example)
* [Creating Custom Evaluators](/how-to-guides/evaluators/create-local-evaluator/create-local-evaluator)

### Related Concepts

* [Agentic Evaluators Overview](/concepts/evaluators/agentic/agentic-overview)
* [Action Completion](/concepts/evaluators/agentic/action-completion)
* [Action Advancement](/concepts/evaluators/agentic/action-advancement)