Skip to content

Commit 8dd0e71

Browse files
ccreutziccreutzi
committed
Move more test points towards more mocks
1 parent 92449b8 commit 8dd0e71

File tree

6 files changed

+335
-223
lines changed

6 files changed

+335
-223
lines changed

‎tests/hopenAIChat.m

Lines changed: 78 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -20,28 +20,11 @@
2020
gpt35Model
2121
end
2222

23-
methods(Test)
24-
% Test methods
25-
function generateAcceptsSingleStringAsInput(testCase,StringInputs)
26-
response = testCase.verifyWarningFree(...
27-
@()generate(testCase.defaultModel,StringInputs));
28-
testCase.verifyClass(response,'string');
29-
testCase.verifyGreaterThan(strlength(response),0);
30-
end
31-
32-
function generateMultipleResponses(testCase)
33-
[~,~,response] = generate(testCase.defaultModel,"What is a cat?",NumCompletions=3);
34-
testCase.verifySize(response.Body.Data.choices,[3,1]);
35-
end
36-
37-
function generateAcceptsMessagesAsInput(testCase)
38-
messages = messageHistory;
39-
messages = addUserMessage(messages, "This should be okay.");
40-
41-
testCase.verifyWarningFree(...
42-
@()generate(testCase.defaultModel,messages));
43-
end
23+
methods (Abstract)
24+
responseMessage
25+
end
4426

27+
methods (Test) % not calling the server
4528
function validConstructorCalls(testCase,ValidConstructorInput)
4629
if isempty(ValidConstructorInput.ExpectedWarning)
4730
chat = testCase.verifyWarningFree(...
@@ -60,17 +43,87 @@ function validConstructorCalls(testCase,ValidConstructorInput)
6043
function fixedSeedFixesResult(testCase)
6144
% Seed is "beta" in OpenAI documentation
6245
% and not reliable at this time.
63-
testCase.assumeFail("disabled since the server is unreliable in honoring the Seed parameter");
46+
import matlab.unittest.constraints.HasField
47+
[sendRequestMock,sendRequestBehaviour] = ...
48+
createMock(testCase, AddedMethods="sendRequest");
49+
testCase.assignOutputsWhen( ...
50+
withAnyInputs(sendRequestBehaviour.sendRequest),...
51+
testCase.responseMessage("Hello"),"This output is unused with Stream=false");
52+
53+
chat = testCase.defaultModel;
54+
chat.sendRequestFcn = @(varargin) sendRequestMock.sendRequest(varargin{:});
55+
56+
generate(chat,"This is okay", "Seed", 2);
57+
58+
calls = testCase.getMockHistory(sendRequestMock);
59+
60+
testCase.verifySize(calls,[1,1]);
61+
sentHistory = calls.Inputs{2};
62+
testCase.assertThat(sentHistory,HasField("seed"));
63+
testCase.verifyEqual(sentHistory.seed,2);
64+
end
65+
66+
function generateOverridesProperties(testCase)
67+
import matlab.unittest.constraints.HasField
68+
[sendRequestMock,sendRequestBehaviour] = ...
69+
createMock(testCase, AddedMethods="sendRequest");
70+
testCase.assignOutputsWhen( ...
71+
withAnyInputs(sendRequestBehaviour.sendRequest),...
72+
testCase.responseMessage("Hello"),"This output is unused with Stream=false");
73+
74+
chat = testCase.defaultModel;
75+
chat.sendRequestFcn = @(varargin) sendRequestMock.sendRequest(varargin{:});
6476

65-
result1 = generate(testCase.defaultModel,"This is okay", "Seed", 2);
66-
result2 = generate(testCase.defaultModel,"This is okay", "Seed", 2);
67-
testCase.verifyEqual(result1,result2);
77+
generate(chat, "Please count from 1 to 10.", Temperature=0, StopSequences="4");
78+
79+
calls = testCase.getMockHistory(sendRequestMock);
80+
81+
testCase.assertSize(calls,[1,1]);
82+
sentHistory = calls.Inputs{2};
83+
testCase.assertThat(sentHistory,HasField("temperature"));
84+
testCase.verifyEqual(sentHistory.temperature, 0);
85+
testCase.assertThat(sentHistory,HasField("stop"));
86+
testCase.verifyEqual(sentHistory.stop, "4");
6887
end
6988

7089
function invalidInputsConstructor(testCase, InvalidConstructorInput)
7190
testCase.verifyError(@()testCase.constructor(InvalidConstructorInput.Input{:}), InvalidConstructorInput.Error);
7291
end
7392

93+
function keyNotFound(testCase)
94+
% to verify the error, we need to unset the environment variable
95+
% OPENAI_API_KEY, if given. Use a fixture to restore the
96+
% value on leaving the test point:
97+
import matlab.unittest.fixtures.EnvironmentVariableFixture
98+
testCase.applyFixture(EnvironmentVariableFixture("OPENAI_API_KEY","dummy"));
99+
unsetenv("OPENAI_API_KEY");
100+
testCase.applyFixture(EnvironmentVariableFixture("AZURE_OPENAI_API_KEY","dummy"));
101+
unsetenv("AZURE_OPENAI_API_KEY");
102+
testCase.verifyError(testCase.constructor, "llms:keyMustBeSpecified");
103+
end
104+
end
105+
106+
methods (Test) % end-to-end, calling the server
107+
function generateAcceptsSingleStringAsInput(testCase,StringInputs)
108+
response = testCase.verifyWarningFree(...
109+
@()generate(testCase.defaultModel,StringInputs));
110+
testCase.verifyClass(response,'string');
111+
testCase.verifyGreaterThan(strlength(response),0);
112+
end
113+
114+
function generateMultipleResponses(testCase)
115+
[~,~,response] = generate(testCase.defaultModel,"What is a cat?",NumCompletions=3);
116+
testCase.verifySize(response.Body.Data.choices,[3,1]);
117+
end
118+
119+
function generateAcceptsMessagesAsInput(testCase)
120+
messages = messageHistory;
121+
messages = addUserMessage(messages, "This should be okay.");
122+
123+
testCase.verifyWarningFree(...
124+
@()generate(testCase.defaultModel,messages));
125+
end
126+
74127
function generateWithStreamFunAndMaxNumTokens(testCase)
75128
sf = @(x) x;
76129
chat = testCase.constructor(StreamFun=sf);
@@ -101,12 +154,6 @@ function generateWithMultipleImages(testCase)
101154
ContainsSubstring("very similar"));
102155
end
103156

104-
function generateOverridesProperties(testCase)
105-
import matlab.unittest.constraints.EndsWithSubstring
106-
text = generate(testCase.defaultModel, "Please count from 1 to 10.", Temperature = 0, StopSequences = "4");
107-
testCase.verifyThat(text, EndsWithSubstring("3, "));
108-
end
109-
110157
function invalidInputsGenerate(testCase, InvalidGenerateInput)
111158
f = openAIFunction("validfunction");
112159
chat = testCase.constructor(Tools=f, APIKey="this-is-not-a-real-key");
@@ -161,18 +208,6 @@ function jsonFormatWithPrompt(testCase)
161208
"string");
162209
end
163210

164-
function keyNotFound(testCase)
165-
% to verify the error, we need to unset the environment variable
166-
% OPENAI_API_KEY, if given. Use a fixture to restore the
167-
% value on leaving the test point:
168-
import matlab.unittest.fixtures.EnvironmentVariableFixture
169-
testCase.applyFixture(EnvironmentVariableFixture("OPENAI_API_KEY","dummy"));
170-
unsetenv("OPENAI_API_KEY");
171-
testCase.applyFixture(EnvironmentVariableFixture("AZURE_OPENAI_API_KEY","dummy"));
172-
unsetenv("AZURE_OPENAI_API_KEY");
173-
testCase.verifyError(testCase.constructor, "llms:keyMustBeSpecified");
174-
end
175-
176211
function toolCallingAndStructuredOutput(testCase)
177212
import matlab.unittest.constraints.HasField
178213

‎tests/hstructuredOutput.m

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,41 @@
77
constructor
88
structuredModel
99
end
10+
11+
methods (Abstract)
12+
responseMessage
13+
end
1014

11-
methods(Test)
15+
methods (Test) % not calling the server
1216
function constructWithStructuredOutput(testCase)
1317
responseFormat = struct("llmReply","This is an example struct");
1418
testCase.verifyWarningFree(@() testCase.constructor(ResponseFormat=responseFormat));
1519
end
1620

21+
function incompleteJSONResponse(testCase)
22+
[sendRequestStub,sendRequestBehaviour] = ...
23+
createMock(testCase, AddedMethods="sendRequest");
24+
testCase.assignOutputsWhen( ...
25+
withAnyInputs(sendRequestBehaviour.sendRequest),...
26+
testCase.responseMessage("{""country"":""Azerbaijan"","),...
27+
"This output is unused with Stream=false");
28+
29+
country = ["USA";"UK"];
30+
capital = ["Washington, D.C.";"London"];
31+
population = [345716792;69203012];
32+
prototype = struct("country",country,"capital",capital,"population",population);
33+
34+
chat = testCase.structuredModel;
35+
chat.sendRequestFcn = @(varargin) sendRequestStub.sendRequest(varargin{:});
36+
37+
testCase.verifyError(@() generate(testCase.structuredModel, ...
38+
"What are the five largest countries whose English names" + ...
39+
" start with the letter A?", ...
40+
ResponseFormat = prototype, MaxNumTokens=3), "llms:apiReturnedIncompleteJSON");
41+
end
42+
end
43+
44+
methods (Test) % calling the server, end-to-end tests
1745
function generateWithStructuredOutput(testCase)
1846
import matlab.unittest.constraints.ContainsSubstring
1947
import matlab.unittest.constraints.StartsWithSubstring
@@ -42,18 +70,6 @@ function generateWithNestedStructs(testCase)
4270
testCase.verifyCompatibleStructs(res,prototype);
4371
end
4472

45-
function incompleteJSONResponse(testCase)
46-
country = ["USA";"UK"];
47-
capital = ["Washington, D.C.";"London"];
48-
population = [345716792;69203012];
49-
prototype = struct("country",country,"capital",capital,"population",population);
50-
51-
testCase.verifyError(@() generate(testCase.structuredModel, ...
52-
"What are the five largest countries whose English names" + ...
53-
" start with the letter A?", ...
54-
ResponseFormat = prototype, MaxNumTokens=3), "llms:apiReturnedIncompleteJSON");
55-
end
56-
5773
function generateWithExplicitSchema(testCase)
5874
import matlab.unittest.constraints.IsSameSetAs
5975
schema = iGetSchema();

‎tests/htoolCalls.m

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,13 @@
77
defaultModel
88
end
99

10-
methods(Test)
10+
methods (Test) % not calling the server
1111
function errorsWhenPassingToolChoiceWithEmptyTools(testCase)
1212
testCase.verifyError(@()generate(testCase.defaultModel,"input", ToolChoice="bla"), "llms:mustSetFunctionsForCall");
1313
end
14+
end
1415

16+
methods (Test) % calling the server, end-to-end tests
1517
function settingToolChoiceWithNone(testCase)
1618
functions = openAIFunction("funName");
1719
chat = testCase.constructor(Tools=functions);

‎tests/tazureChat.m

Lines changed: 43 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
gpt35Model = azureChat(DeploymentID="gpt-35-turbo-16k-0613");
2020
end
2121

22-
methods(Test)
22+
methods (Test) % not calling the server
2323
function constructChatWithAllNVP(testCase)
2424
deploymentID = "hello";
2525
functions = openAIFunction("funName");
@@ -41,21 +41,13 @@ function constructChatWithAllNVP(testCase)
4141
testCase.verifyEqual(chat.PresencePenalty, presenceP);
4242
end
4343

44-
function doGenerateUsingSystemPrompt(testCase)
45-
testCase.assumeTrue(isenv("AZURE_OPENAI_API_KEY"),"end-to-end test requires environment variables AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT, and AZURE_OPENAI_DEPLOYMENT.");
46-
chat = azureChat("You are a helpful assistant");
47-
response = testCase.verifyWarningFree(@() generate(chat,"Hi"));
48-
testCase.verifyClass(response,'string');
49-
testCase.verifyGreaterThan(strlength(response),0);
50-
end
51-
5244
function sendsSystemPrompt(testCase)
5345
import matlab.unittest.constraints.HasField
5446
[sendRequestMock,sendRequestBehaviour] = ...
5547
createMock(testCase, AddedMethods="sendRequest");
5648
testCase.assignOutputsWhen( ...
5749
withAnyInputs(sendRequestBehaviour.sendRequest),...
58-
iResponseMessage("Hello"),"This output is unused with Stream=false");
50+
testCase.responseMessage("Hello"),"This output is unused with Stream=false");
5951

6052
chat = testCase.constructor("You are a helpful assistant");
6153
chat.sendRequestFcn = @(varargin) sendRequestMock.sendRequest(varargin{:});
@@ -75,6 +67,36 @@ function sendsSystemPrompt(testCase)
7567
testCase.verifyEqual(response,"Hello");
7668
end
7769

70+
function endpointNotFound(testCase)
71+
% to verify the error, we need to unset the environment variable
72+
% AZURE_OPENAI_ENDPOINT, if given. Use a fixture to restore the
73+
% value on leaving the test point
74+
import matlab.unittest.fixtures.EnvironmentVariableFixture
75+
testCase.applyFixture(EnvironmentVariableFixture("AZURE_OPENAI_ENDPOINT","dummy"));
76+
unsetenv("AZURE_OPENAI_ENDPOINT");
77+
testCase.verifyError(@()azureChat, "llms:endpointMustBeSpecified");
78+
end
79+
80+
function deploymentNotFound(testCase)
81+
% to verify the error, we need to unset the environment variable
82+
% AZURE_OPENAI_DEPLOYMENT, if given. Use a fixture to restore the
83+
% value on leaving the test point
84+
import matlab.unittest.fixtures.EnvironmentVariableFixture
85+
testCase.applyFixture(EnvironmentVariableFixture("AZURE_OPENAI_DEPLOYMENT","dummy"));
86+
unsetenv("AZURE_OPENAI_DEPLOYMENT");
87+
testCase.verifyError(@()azureChat, "llms:deploymentMustBeSpecified");
88+
end
89+
end
90+
91+
methods (Test) % end-to-end, calling the server
92+
function doGenerateUsingSystemPrompt(testCase)
93+
testCase.assumeTrue(isenv("AZURE_OPENAI_API_KEY"),"end-to-end test requires environment variables AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT, and AZURE_OPENAI_DEPLOYMENT.");
94+
chat = azureChat("You are a helpful assistant");
95+
response = testCase.verifyWarningFree(@() generate(chat,"Hi"));
96+
testCase.verifyClass(response,'string');
97+
testCase.verifyGreaterThan(strlength(response),0);
98+
end
99+
78100
function generateMultipleResponses(testCase)
79101
chat = azureChat;
80102
[~,~,response] = generate(chat,"What is a cat?",NumCompletions=3);
@@ -131,13 +153,6 @@ function generateWithMultipleImages(testCase)
131153
testCase.verifyThat(text,ContainsSubstring("same") | ContainsSubstring("identical"));
132154
end
133155

134-
function generateOverridesProperties(testCase)
135-
import matlab.unittest.constraints.EndsWithSubstring
136-
chat = azureChat;
137-
text = generate(chat, "Please count from 1 to 10.", Temperature = 0, StopSequences = "4");
138-
testCase.verifyThat(text, EndsWithSubstring("3, "));
139-
end
140-
141156
function shortErrorForBadEndpoint(testCase)
142157
chat = azureChat(Endpoint="https://nobodyhere.whatever/");
143158
caught = false;
@@ -170,27 +185,19 @@ function specialErrorForUnsupportedResponseFormat(testCase)
170185
ResponseFormat=struct("number",1)), ...
171186
"llms:noStructuredOutputForAzureDeployment");
172187
end
188+
end
173189

174-
function endpointNotFound(testCase)
175-
% to verify the error, we need to unset the environment variable
176-
% AZURE_OPENAI_ENDPOINT, if given. Use a fixture to restore the
177-
% value on leaving the test point
178-
import matlab.unittest.fixtures.EnvironmentVariableFixture
179-
testCase.applyFixture(EnvironmentVariableFixture("AZURE_OPENAI_ENDPOINT","dummy"));
180-
unsetenv("AZURE_OPENAI_ENDPOINT");
181-
testCase.verifyError(@()azureChat, "llms:endpointMustBeSpecified");
182-
end
183-
184-
function deploymentNotFound(testCase)
185-
% to verify the error, we need to unset the environment variable
186-
% AZURE_OPENAI_DEPLOYMENT, if given. Use a fixture to restore the
187-
% value on leaving the test point
188-
import matlab.unittest.fixtures.EnvironmentVariableFixture
189-
testCase.applyFixture(EnvironmentVariableFixture("AZURE_OPENAI_DEPLOYMENT","dummy"));
190-
unsetenv("AZURE_OPENAI_DEPLOYMENT");
191-
testCase.verifyError(@()azureChat, "llms:deploymentMustBeSpecified");
190+
methods
191+
function msg = responseMessage(~,txt)
192+
% minimal structure replacing the real matlab.net.http.ResponseMessage() in our mocks
193+
msg = struct(...
194+
StatusCode="OK",...
195+
Body=struct(...
196+
Data=struct(...
197+
choices=struct(...
198+
message=struct(...
199+
content=txt)))));
192200
end
193-
194201
end
195202
end
196203

@@ -529,14 +536,3 @@ function deploymentNotFound(testCase)
529536
function apiVersions = iGetAPIVersions()
530537
apiVersions = cellstr(llms.azure.apiVersions);
531538
end
532-
533-
function msg = iResponseMessage(txt)
534-
% minimal structure replacing the real matlab.net.http.ResponseMessage() in our mocks
535-
msg = struct(...
536-
StatusCode="OK",...
537-
Body=struct(...
538-
Data=struct(...
539-
choices=struct(...
540-
message=struct(...
541-
content=txt)))));
542-
end

0 commit comments

Comments
 (0)