1// 1. 智能体创建config(端或服务端配置)里配置 enable_visual为true；配置服务端开启多模态功能；通过screen_width、screen_height 属性指定生成的图片分辨率；
2#define JSON_CONFIG_TEMPLATE_VISUAL "{\"app_id\": \"%s\", \"config\" : \"{\\\"llm\\\" : \\\"%s\\\", \\\"llm_token\\\" : \\\"no\\\", \\\"enable_visual\\\" : \\\"%s\\\", \\\"screen_width\\\" : \\\"640\\\", \\\"screen_height\\\" : \\\"480\\\", \\\"dfda\\\" : \\\"true\\\",\\\"remote_music_player\\\" : \\\"true\\\", \\\"rtc_ac\\\": \\\"pcmu\\\", \\\"lang\\\" : \\\"%s\\\"}\", \"quick_start\": true}"
3
4void brtc_init(void)
5{
6    BaiduChatAgentEvent events = {
7        .onError = onErrorCallback,
8        .onCallStateChange = onCallStateChangeCallback,
9        .onConnectionStateChange = onConnectionStateChangeCallback,
10        .onUserAsrSubtitle = onUserAsrSubtitleCallback,
11        .onFunctionCall = onFunctionCall,
12        .onMediaSetup = onMediaSetup,
13        .onAIAgentSubtitle = onAIAgentSubtitle,
14        .onAIAgentSpeaking = onAIAgentSpeaking,
15        .onAudioPlayerOp = onAudioPlayerOp,
16        .onAudioData = onAudioData,
17        .onVideoData = onVideoData,  // 2.设置视频、图片接收回调, 生成的图片从该接口下发；
18        .onLicenseResult = onLicenseResult,
19        ... ... 
20        .onMediaGenerateResult = onMediaGenerateResult, // 3.设置图片生成结果回调
21    };
22    
23    AgentEngineParams agentParams;
24    memset(&agentParams, 0, sizeof(agentParams));
25    setUserParameters(&agentParams);
26    // 初始化智能体引擎
27    int result = baidu_chat_agent_engine_init(engine, &agentParams);
28    ... 
29    baidu_chat_agent_engine_call(engine);
30}
31
32void setUserParameters(AgentEngineParams *params)
33{
34    strncpy(params->agent_platform_url, g_platform_host, sizeof(params->agent_platform_url) - 1);
35    ...  ... 
36    params->AudioInChannel = 1;
37    params->AudioInFrequency = 8000;
38    if(g_enable_visual || g_enable_image_generate) {
39        params->enable_video = true; // 3. 多模态场景（生图、视觉）开启视频通道
40    }
41    params->region = region;
42}

1// 1. 自定义生图prompt， 可参考这个修改自定义的prompt, 注意转义符号；
2static const char* object_media_generate_prompt = 
3    "要求生成精致的黑白简笔画。严格只用纯黑线与纯白背景，无灰度、无填色、无阴影、无高光、无纹理、无噪点。线条要求： \
4    统一且细的单一线宽，干净、连续、锐利；允许在主体内部加入更多**同线宽的细节线条**（例如：发丝、衣褶、须眉、纹饰等）\
5    以增加精致感，但所有细节必须由与轮廓相同的单一线宽构成。构图需简洁、留白充分、主体明确。\\\\n\\\\n";
6
7static char media_generate_prompt_text[4096]; 
8// 2. 生图、视觉 prompt更新消息格式定义，注意括号内为 \\\ 三斜扛转义；
9static const char g_updata_media_generate_prompt_cmd[] = "%s{\\\"model_type\\\":\\\"%s\\\",\\\"prompt\\\":\\\"%s\\\"}";
10
11void update_media_generate_prompt(char *prompt) {
12    int prompt_len = prompt ? strlen(prompt) : 0;
13    int total_len = prompt_len + strlen(g_updata_media_generate_prompt_cmd) + 64;
14    char *update_prompt_cmd = (char *)custom_malloc_psram(total_len);
15
16    if (update_prompt_cmd) {
17        snprintf(update_prompt_cmd, total_len, g_updata_vision_prompt_cmd,
18                    AGENT_EVENT_UPDATE_SYSTEM_PROMPT, "3", prompt);
19    } else {
20        os_printf("update_media_generate_prompt malloc failed\r\n");
21        return;
22    }
23    strncpy(media_generate_prompt_text, update_prompt_cmd, sizeof(media_generate_prompt_text) -1);
24    custom_free_psram(update_prompt_cmd);
25}
26
27void agent_update_prompt(void * text) {
28    baidu_chat_agent_engine_send_event_to_agent(g_engine, (char *)text);
29}
30
31// 3. 更新视觉prompt， 3： 更新生图prompt; 0: 为恢复系统prompt;
32void update_prompt(const char *prompt_mode) {
33    int mode = os_atoi(prompt_mode);
34    os_memset(media_generate_prompt_text, 0, sizeof(media_generate_prompt_text) -1);
35    if (mode == 3) {
36        update_media_generate_prompt(object_vision_prompt);
37    } else if (mode == 0) {
38        // reset media generate prompt
39        update_media_generate_prompt("");
40    }
41
42    os_printf("update prompt text:%s\r\n", media_generate_prompt_text);
43    os_task_create("brtc_task_update_prompt", agent_update_prompt, (void*)media_generate_prompt_text, OS_TASK_PRIORITY_NORMAL, 0, NULL, 16 * 1024);
44}

1void onVideoData(const uint8_t *data, size_t len, RtcImageType imgtype, int width, int height)
2{
3    BRTC_LOG("FrameReceived video data of length: %d, width: %d, height: %d\n", len, width, height);
4    if (imgtype == RTC_IMAGE_TYPE_JPEG) {
5#ifdef DUMP_VIDEO_FRAME
6        if (len > 0) {
7            char filename[64] = {0};
8            os_sprintf(filename, "0:video_frame_%04d.jpeg", (uint32_t)os_jiffies() % 9999);
9            dumpVideoFrame(filename, data, len);
10        }
11#endif        
12        // render to display
13        jpeg_photo_renderer(data, len, 320, 240);
14    }
15}

1void onMediaGenerateResult(const char* result) {
2    if (result && strlen(result) > 0) {
3        BRTC_LOG("onMediaGenerateResult content: %s\n", result);
4    }
5}