promptfooconfig.predict.yaml

description: |
  /predict endpoint eval — full pipeline: categorize → blend → pools →
  cache → personalization. Hits the running uvicorn server on :8900.
  Per-case llm-rubric checks the surfaced `words` array makes sense for
  an AAC kid (kid-safe, lowercase, single words, grammatically following).

  Run:
    # 1. Start the backend in another terminal:  ./start.sh
    # 2. Then:
    OPENROUTER_KEY=<your_key> \
    npx promptfoo@latest eval -c promptfooconfig.predict.yaml

prompts:
  - id: predict-body
    raw: |
      {"sentence": {{sentence_json}}, "n": 8}

providers:
  - id: http://localhost:8900/predict
    config:
      url: http://localhost:8900/predict
      method: POST
      headers:
        Content-Type: application/json
      body:
        sentence: '{{sentence_json}}'
        n: 8
      transformResponse: |
        json.words ? json.words.join(", ") : JSON.stringify(json)

defaultTest:
  options:
    provider:
      id: openai:chat:gpt-4o-mini
      config:
        apiBaseUrl: https://openrouter.ai/api/v1
        apiKeyEnvar: OPENROUTER_KEY
        headers:
          HTTP-Referer: http://localhost
          X-Title: aac-rubric
  assert:
    - type: javascript
      value: |
        // Output is a comma-separated list of 8 lowercase single tokens.
        const words = output.split(",").map(s => s.trim()).filter(Boolean);
        if (words.length < 6) return { pass: false, reason: `only ${words.length} words returned` };
        for (const w of words) {
          if (w !== w.toLowerCase()) return { pass: false, reason: `non-lowercase: ${w}` };
          if (!/^[a-z']+$/.test(w)) return { pass: false, reason: `non-alpha word: ${w}` };
        }
        const set = new Set(words);
        if (set.size < words.length) return { pass: false, reason: "duplicates in output" };
        return true;

tests:
  - description: 'empty sentence — sentence-starters'
    vars:
      sentence_json: "[]"
    assert:
      - type: llm-rubric
        value: |
          The output is a comma-separated list of single lowercase words.
          For an empty sentence, MOST words MUST be plausible sentence-starters
          for an 11-year-old AAC user: things like "i", "you", "we", "let's",
          "can", "where", "what", "the", "my", "hi", "yes", "no".
          NO unusual or overly complex words.

  - description: 'bare "i" — verbs and "am" / "want"'
    vars:
      sentence_json: '["i"]'
    assert:
      - type: llm-rubric
        value: |
          Each word MUST be one that can grammatically follow "i" in a kid sentence:
          base verbs (want, need, like, love, hate, see, have, play, feel),
          contractions/auxiliaries (am, will, can), or short connectors.
          NO bare nouns ("i pizza" is wrong). NO -ing verbs without "am".
          At least 4 of the 8 should be common kid verbs.

  - description: 'sentence "i want"'
    vars:
      sentence_json: '["i", "want"]'
    assert:
      - type: llm-rubric
        value: |
          Each word MUST grammatically follow "i want": "to" (very common),
          "a", "the", "my", "more", "some", or a noun (food/thing/game).
          The list SHOULD include "to". MUST NOT include another verb in
          base form (e.g. "play" without "to" first is awkward but acceptable
          since AAC kids stack words; reject only if MAJORITY are bare verbs).

  - description: 'sentence "i want to"'
    vars:
      sentence_json: '["i", "want", "to"]'
    assert:
      - type: llm-rubric
        value: |
          Every word MUST be a base-form verb that an 11-year-old says:
          play, eat, go, see, have, build, watch, read, draw, sleep, etc.
          NO -ing verbs. NO nouns. NO function words like "to"/"a"/"the".

  - description: "sentence \"i'm super\""
    vars:
      sentence_json: "[\"i'm\", \"super\"]"
    assert:
      - type: llm-rubric
        value: |
          Every word MUST be an adjective Adrian could pair with "super":
          tired, cool, hungry, scared, happy, mad, fast, strong, awesome,
          excited, bored, etc. NO verbs. NO nouns. NO "super" itself.

  - description: '"i''m super tired" — sentence-end / connectors'
    vars:
      sentence_json: "[\"i'm\", \"super\", \"tired\"]"
    assert:
      - type: llm-rubric
        value: |
          The list MUST be dominated by connectors/sentence-end markers:
          "and", "but", "so", "now", "today", "too", or possibly an adverb.
          MUST NOT include game names, food, or new objects (no second clause-content yet).
          Reject if any word looks like a fresh subject/object.

  - description: '"i love minecraft" — conjunction next, NOT another game'
    vars:
      sentence_json: '["i", "love", "minecraft"]'
    assert:
      - type: llm-rubric
        value: |
          The list MUST include "and" (very high priority for chaining).
          The list MUST NOT include another game name (roblox, sonic, fortnite,
          chess, etc) — that would be ungrammatical or unnatural after a
          completed S-V-O.
          Acceptable: and, but, so, with, too, now, today, please.

  - description: '"i don''t like" — object slot'
    vars:
      sentence_json: "[\"i\", \"don't\", \"like\"]"
    assert:
      - type: llm-rubric
        value: |
          Each word MUST be something a kid could DISLIKE: a game, food,
          activity, person, or demonstrative ("this", "that", "it").
          MUST NOT include another negation ("not", "no", "don't").
          MUST NOT be only verbs.

  - description: 'sentence "the creeper is"'
    vars:
      sentence_json: '["the", "creeper", "is"]'
    assert:
      - type: llm-rubric
        value: |
          Each word MUST grammatically follow "is": -ing verbs (coming,
          chasing, exploding, attacking), adjectives (scary, green, fast,
          dangerous), or "super" / "really" intensifiers.
          NO base-form verbs (no "run", "jump"). NO bare nouns.

  - description: "sentence \"let's go to the\""
    vars:
      sentence_json: "[\"let's\", \"go\", \"to\", \"the\"]"
    assert:
      - type: llm-rubric
        value: |
          Each word MUST be a place or thing-as-destination an 11-year-old
          might pick: park, beach, store, mall, library, school, kitchen,
          bathroom, pool, game, movies, etc.
          MUST NOT include verbs, "the" again, or "a".

  - description: 'sentence "where is"'
    vars:
      sentence_json: '["where", "is"]'
    assert:
      - type: llm-rubric
        value: |
          Each word MUST be a noun/pronoun/determiner that fits "where is X":
          mom, dad, the (+noun), my, it, that, this, your, our, plus people
          / things / places. NO verbs. NO conjunctions.

  - description: 'sentence "can we"'
    vars:
      sentence_json: '["can", "we"]'
    assert:
      - type: llm-rubric
        value: |
          Each word MUST be a base verb a kid would request: play, go, eat,
          watch, build, make, have, see, get, try.
          NO past-tense or -ing forms. NO nouns.

  - description: 'sentence "i feel"'
    vars:
      sentence_json: '["i", "feel"]'
    assert:
      - type: llm-rubric
        value: |
          Each word MUST be an adjective or feeling word: happy, sad, mad,
          tired, hungry, sick, good, bad, scared, super (intensifier),
          really, so. NO verbs. NO nouns.

  - description: 'sentence "you are"'
    vars:
      sentence_json: '["you", "are"]'
    assert:
      - type: llm-rubric
        value: |
          Each word MUST follow "you are" naturally: adjectives (cool, mean,
          funny, awesome, super), -ing verbs (going, coming, playing), or
          role nouns (mom, dad, my, the +noun).
          NO base verbs in raw form ("you are play" is wrong).

  - description: '"and" mid-sentence — new clause start'
    vars:
      sentence_json: '["i", "love", "chess", "and"]'
    assert:
      - type: llm-rubric
        value: |
          A new clause is beginning. Words SHOULD be subjects ("i", "we",
          "you", "it"), or another game/thing being added to the love-list,
          or a new verb-phrase opener.
          MUST NOT include another conjunction ("and", "but", "or", "so").

  - description: 'question form "what do you"'
    vars:
      sentence_json: '["what", "do", "you"]'
    assert:
      - type: llm-rubric
        value: |
          Each word MUST be a base-form verb that completes a kid question:
          want, like, need, see, have, do, think, mean, say.
          NO nouns. NO -ing verbs.

  - description: 'sentence "my mom is"'
    vars:
      sentence_json: '["my", "mom", "is"]'
    assert:
      - type: llm-rubric
        value: |
          Each word MUST be an adjective, -ing verb, or role noun that fits
          after "is": cool, busy, sleeping, working, here, home, the +noun,
          super (intensifier).
          NO base verbs.

  - description: 'empty followed by Adrian-favorite proper-noun bias'
    vars:
      sentence_json: '["i", "love"]'
    assert:
      - type: llm-rubric
        value: |
          The list MUST include at least one game or character Adrian likes:
          minecraft, roblox, chess, sonic, batman, OR a noun like "you",
          "mom", "dad", or food/animal.
          NO bare verbs (the slot after "love" wants a thing, not an action).