Skip to content

Commit f0c6e84

Browse files
songgotsonggot
committed
[API] Add ML LXM Service API(internal) for large model interactons
This commit introduces the ML LXM Service API, a new C API designed to facilitate interactions with large-scale models such as Large Language Models (LLMs) Signed-off-by: hyunil park <[email protected]>
1 parent 71aaab1 commit f0c6e84

File tree

3 files changed

+553
-1
lines changed

3 files changed

+553
-1
lines changed

‎c/include/ml-lxm-service-internal.h‎

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
/* SPDX-License-Identifier: Apache-2.0 */
2+
/**
3+
* @file ml-lxm-service-internal.h
4+
* @date 23 JULY 2025
5+
* @brief Machine Learning LXM(LLM, LVM, etc.) Service API
6+
* @see https://github.com/nnstreamer/api
7+
* @author Hyunil Park <[email protected]>
8+
* @bug No known bugs except for NYI items
9+
*/
10+
11+
/**
12+
* @example sample_lxm_service.c
13+
* @brief Sample application demonstrating ML LXM Service API usage
14+
*
15+
* This sample shows how to:
16+
* - Create and configure an LXM session
17+
* - Build prompts with text and instructions
18+
* - Generate streaming responses with custom options
19+
* - Handle token callbacks for real-time processing
20+
*
21+
* Configuration file example (config.json):
22+
* @code
23+
* {
24+
* "single" :
25+
* {
26+
* "framework" : "flare",
27+
* "model" : ["sflare_if_4bit_3b.bin"],
28+
* "adapter" : ["history_lora.bin"],
29+
* "custom" : "tokenizer_path:tokenizer.json,backend:CPU,output_size:1024,model_type:3B,data_type:W4A32",
30+
* "invoke_dynamic" : "true",
31+
* }
32+
* }
33+
* @endcode
34+
*
35+
* Basic usage workflow:
36+
* @code
37+
* // 1. Create session with callback
38+
* ml_lxm_session_h session;
39+
* ml_lxm_session_create("/path/to/config.json", NULL, token_handler, NULL, &session);
40+
*
41+
* // 2. Create prompt
42+
* ml_lxm_prompt_h prompt;
43+
* ml_lxm_prompt_create(&prompt);
44+
* ml_lxm_prompt_append_text(prompt, "Hello AI");
45+
*
46+
* // 3. Generate response with options (callback is already set during session creation)
47+
* ml_option_h options = NULL;
48+
* ml_option_create(&options);
49+
* ml_option_set(options, "temperature", g_strdup_printf("%f", 1.0), g_free);
50+
* ml_option_set(options, "max_tokens", g_strdup_printf("%zu", (size_t)50), g_free);
51+
* ml_lxm_session_respond(session, prompt, options);
52+
* ml_option_destroy(options);
53+
*
54+
* // 4. Cleanup
55+
* ml_lxm_prompt_destroy(prompt);
56+
* ml_lxm_session_destroy(session);
57+
* @endcode
58+
*
59+
* Complete example with token callback:
60+
* @code
61+
* #include "ml-lxm-service-internal.h"
62+
* #include <iostream>
63+
*
64+
* static void token_handler(ml_service_event_e event,
65+
* ml_information_h event_data,
66+
* void *user_data);
67+
*
68+
* int main() {
69+
* ml_lxm_session_h session = NULL;
70+
* ml_lxm_prompt_h prompt = NULL;
71+
* int ret;
72+
*
73+
* // Check availability first
74+
* ml_lxm_availability_e status;
75+
* ret = ml_lxm_check_availability(&status);
76+
* if (ret != ML_ERROR_NONE || status != ML_LXM_AVAILABILITY_AVAILABLE) {
77+
* std::cout << "LXM service not available" << std::endl;
78+
* return -1;
79+
* }
80+
*
81+
* // 1. Create session with config, instructions, and callback
82+
* ret = ml_lxm_session_create("/path/to/config.json", "You are a helpful AI assistant", token_handler, NULL, &session);
83+
* if (ret != ML_ERROR_NONE) {
84+
* std::cout << "Failed to create session" << std::endl;
85+
* return -1;
86+
* }
87+
*
88+
* // 2. Create prompt
89+
* ret = ml_lxm_prompt_create(&prompt);
90+
* if (ret != ML_ERROR_NONE) {
91+
* std::cout << "Failed to create prompt" << std::endl;
92+
* ml_lxm_session_destroy(session);
93+
* return -1;
94+
* }
95+
*
96+
* // Add text to prompt
97+
* ret = ml_lxm_prompt_append_text(prompt, "Explain quantum computing in simple terms");
98+
* if (ret != ML_ERROR_NONE) {
99+
* std::cout << "Failed to append text to prompt" << std::endl;
100+
* ml_lxm_prompt_destroy(prompt);
101+
* ml_lxm_session_destroy(session);
102+
* return -1;
103+
* }
104+
*
105+
* // 3. Generate response with custom options
106+
* ml_option_h options = NULL;
107+
* ml_option_create(&options);
108+
* ml_option_set(options, "temperature", g_strdup_printf("%f", 1.2), g_free);
109+
* ml_option_set(options, "max_tokens", g_strdup_printf("%zu", (size_t)128), g_free);
110+
*
111+
* std::cout << "AI Response: ";
112+
* ret = ml_lxm_session_respond(session, prompt, options);
113+
* ml_option_destroy(options);
114+
* if (ret != ML_ERROR_NONE) {
115+
* std::cout << "Failed to generate response" << std::endl;
116+
* }
117+
* std::cout << std::endl;
118+
*
119+
* // 4. Cleanup
120+
* ml_lxm_prompt_destroy(prompt);
121+
* ml_lxm_session_destroy(session);
122+
*
123+
* return 0;
124+
* }
125+
*
126+
* static void token_handler(ml_service_event_e event,
127+
* ml_information_h event_data,
128+
* void *user_data) {
129+
* ml_tensors_data_h data = NULL;
130+
* void *_raw = NULL;
131+
* size_t _size = 0;
132+
* int ret;
133+
*
134+
* switch (event) {
135+
* case ML_SERVICE_EVENT_NEW_DATA:
136+
* if (event_data != NULL) {
137+
* ret = ml_information_get(event_data, "data", &data);
138+
* if (ret == ML_ERROR_NONE) {
139+
* ret = ml_tensors_data_get_tensor_data(data, 0U, &_raw, &_size);
140+
* if (ret == ML_ERROR_NONE && _raw != NULL && _size > 0) {
141+
* std::cout.write(static_cast<const char *>(_raw), _size);
142+
* std::cout.flush();
143+
* }
144+
* }
145+
* }
146+
* break;
147+
* default:
148+
* break;
149+
* }
150+
* }
151+
* @endcode
152+
*/
153+
154+
#ifndef __ML_LXM_SERVICE_INTERNAL_H__
155+
#define __ML_LXM_SERVICE_INTERNAL_H__
156+
157+
#include <stdlib.h>
158+
#include <ml-api-service.h>
159+
#ifdef __cplusplus
160+
extern "C"
161+
{
162+
#endif
163+
164+
/**
165+
* @brief Enumeration for LXM service availability status.
166+
*/
167+
typedef enum
168+
{
169+
ML_LXM_AVAILABILITY_AVAILABLE = 0,
170+
ML_LXM_AVAILABILITY_DEVICE_NOT_ELIGIBLE,
171+
ML_LXM_AVAILABILITY_SERVICE_DISABLED,
172+
ML_LXM_AVAILABILITY_MODEL_NOT_READY,
173+
ML_LXM_AVAILABILITY_UNKNOWN
174+
} ml_lxm_availability_e;
175+
176+
/**
177+
* @brief Checks LXM service availability.
178+
* @param[out] status Current availability status.
179+
* @return ML_ERROR_NONE on success, error code otherwise.
180+
*/
181+
int ml_lxm_check_availability (ml_lxm_availability_e * status);
182+
183+
/**
184+
* @brief A handle for lxm session.
185+
*/
186+
typedef void *ml_lxm_session_h;
187+
188+
/**
189+
* @brief Creates an LXM session with mandatory callback.
190+
* @param[in] config_path Path to configuration file.
191+
* @param[in] instructions Initial instructions (optional).
192+
* @param[in] callback Callback function for session events (mandatory).
193+
* @param[in] user_data User data to be passed to the callback.
194+
* @param[out] session Session handle.
195+
* @return ML_ERROR_NONE on success.
196+
* @note The callback parameter is mandatory and will be set during session creation.
197+
*/
198+
int ml_lxm_session_create (const char *config_path, const char *instructions, ml_service_event_cb callback, void *user_data, ml_lxm_session_h * session);
199+
200+
/**
201+
* @brief Destroys an LXM session.
202+
* @param[in] session Session handle.
203+
* @return ML_ERROR_NONE on success.
204+
*/
205+
int ml_lxm_session_destroy (ml_lxm_session_h session);
206+
207+
/**
208+
* @brief A handle for lxm prompt.
209+
*/
210+
typedef void *ml_lxm_prompt_h;
211+
212+
/**
213+
* @brief Creates a prompt object.
214+
* @param[out] prompt Prompt handle.
215+
* @return ML_ERROR_NONE on success.
216+
*/
217+
int ml_lxm_prompt_create (ml_lxm_prompt_h * prompt);
218+
219+
/**
220+
* @brief Appends text to a prompt.
221+
* @param[in] prompt Prompt handle.
222+
* @param[in] text Text to append.
223+
* @return ML_ERROR_NONE on success.
224+
*/
225+
int ml_lxm_prompt_append_text (ml_lxm_prompt_h prompt, const char *text);
226+
227+
/**
228+
* @brief Appends an instruction to a prompt.
229+
* @param[in] prompt Prompt handle.
230+
* @param[in] instruction Instruction to append.
231+
* @return ML_ERROR_NONE on success.
232+
*/
233+
int ml_lxm_prompt_append_instruction (ml_lxm_prompt_h prompt, const char *instruction);
234+
235+
/**
236+
* @brief Destroys a prompt object.
237+
* @param[in] prompt Prompt handle.
238+
* @return ML_ERROR_NONE on success.
239+
*/
240+
int ml_lxm_prompt_destroy (ml_lxm_prompt_h prompt);
241+
242+
/**
243+
* @brief Sets runtime instructions for a session.
244+
* @param[in] session Session handle.
245+
* @param[in] instructions New instructions.
246+
* @return ML_ERROR_NONE on success.
247+
*/
248+
int ml_lxm_session_set_instructions (ml_lxm_session_h session, const char *instructions);
249+
250+
251+
/**
252+
* @brief Generates an token-streamed response.
253+
* @param[in] session Session handle.
254+
* @param[in] prompt Prompt handle.
255+
* @param[in] options Generation parameters (ml_option_h).
256+
* @return ML_ERROR_NONE on success.
257+
* @note The callback should be set using ml_lxm_session_set_event_cb() before calling this function.
258+
*/
259+
int ml_lxm_session_respond (ml_lxm_session_h session, ml_lxm_prompt_h prompt, ml_option_h options);
260+
261+
#ifdef __cplusplus
262+
}
263+
#endif
264+
#endif
265+
/* __ML_LXM_SERVICE_INTERNAL_H__ */

‎c/src/meson.build‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
nns_capi_common_srcs = files('ml-api-common.c', 'ml-api-inference-internal.c')
22
nns_capi_single_srcs = files('ml-api-inference-single.c')
33
nns_capi_pipeline_srcs = files('ml-api-inference-pipeline.c')
4-
nns_capi_service_srcs = files('ml-api-service.c', 'ml-api-service-extension.c', 'ml-api-service-agent-client.c')
4+
nns_capi_service_srcs = files('ml-api-service.c', 'ml-api-service-extension.c', 'ml-api-service-agent-client.c', 'ml-lxm-service.c')
55

66
if support_nnstreamer_edge
77
nns_capi_service_srcs += files('ml-api-service-query.c')

0 commit comments

Comments
 (0)