Enable QWen VL video preprocess

README.md

@@ -160,6 +160,13 @@ image_data = ov.Tensor(image_data)
 
 prompt = "Can you describe the image?"
 result = pipe.generate(prompt, image=image_data, max_new_tokens=100)
+
+# To input multiple images, use 'images='
+# result = pipe.generate(prompt, images=[image_data], max_new_tokens=100)
+
+# To input videos frames, use 'videos='
+# result = pipe.generate(prompt, videos=[frames_data], max_new_tokens=100)
+
 print(result.texts[0])
 ```
 
@@ -181,6 +188,12 @@ int main(int argc, char* argv[]) {
         ov::genai::image(rgb),
         ov::genai::max_new_tokens(100)
     ) << '\n';
+
+    // To input multiple images, use 'images'
+    // pipe.generate(prompt, ov::genai::images(std::vector<ov::Tensor>{rgb}), ov::genai::max_new_tokens(100));
+
+    // To input videos frames, use 'videos'
+    // pipe.generate(prompt, ov::genai::videos(std::vector<ov::Tensor>{frames}), ov::genai::max_new_tokens(100));
 }
 ```
 

src/cpp/include/openvino/genai/continuous_batching_pipeline.hpp

@@ -165,7 +165,16 @@ class OPENVINO_GENAI_EXPORTS ContinuousBatchingPipeline {
     /// @param request_id must be unique for every add_request() call.
     GenerationHandle add_request(uint64_t request_id, const ov::Tensor& input_ids, const ov::genai::GenerationConfig& sampling_params);
     GenerationHandle add_request(uint64_t request_id, const std::string& prompt, const ov::genai::GenerationConfig& sampling_params);
-    GenerationHandle add_request(uint64_t request_id, const std::string& prompt, const std::vector<ov::Tensor>& images, const ov::genai::GenerationConfig& sampling_params);
+    GenerationHandle add_request(uint64_t request_id,
+                                 const std::string& prompt,
+                                 const std::vector<ov::Tensor>& images,
+                                 const ov::genai::GenerationConfig& sampling_params);
+
+    GenerationHandle add_request(uint64_t request_id,
+                                 const std::string& prompt,
+                                 const std::vector<ov::Tensor>& images,
+                                 const std::vector<ov::Tensor>& videos,
+                                 const ov::genai::GenerationConfig& sampling_params);
 
     void step();
 
@@ -179,6 +188,14 @@ class OPENVINO_GENAI_EXPORTS ContinuousBatchingPipeline {
              const std::vector<std::vector<ov::Tensor>>& images,
              const std::vector<GenerationConfig>& sampling_params,
              const StreamerVariant& streamer=std::monostate{});
+
+    std::vector<VLMDecodedResults> generate(
+            const std::vector<std::string>& prompts,
+            const std::vector<std::vector<ov::Tensor>>& images,
+            const std::vector<std::vector<ov::Tensor>>& videos,
+            const std::vector<GenerationConfig>& sampling_params,
+            const StreamerVariant& streamer=std::monostate{});
+
     /**
     * @brief start chat with keeping history in kv cache.
     * @param system_message optional system message.

src/cpp/include/openvino/genai/visual_language/pipeline.hpp

@@ -97,7 +97,26 @@ class OPENVINO_GENAI_EXPORTS VLMPipeline {
     /// To disable it for non-chat mode, please, use custom_chat_template eq "" or set generation_config.apply_chat_template to false.
     VLMDecodedResults generate(
         const std::string& prompt,
-        const std::vector<ov::Tensor>& rgbs,
+        const std::vector<ov::Tensor>& images,
+        const GenerationConfig& generation_config,
+        const StreamerVariant& streamer
+    );
+
+    /// @brief Generate a response given a prompt and any number of
+    /// uint8 RGB images with [NHWC] or [HWC] layout.
+    /// Or uint8 RGB video frames with [NHWC] layout, first dim means frames number.
+    /// @param prompt A prompt to respond to.
+    /// @param images Images to be prepended to a prompt.
+    /// @param videos Multiple videos, each providing multiple frames, to be prepended to a prompt.
+    /// @param generation_config A config to follow for text generation.
+    /// @param streamer A streamer to acquire intermediate result.
+    /// @return A string generated by a model.
+    /// chat_template will be applied to the prompt, run pipe.set_chat_template(custom_chat_template) to update it.
+    /// To disable it for non-chat mode, please, use custom_chat_template eq "" or set generation_config.apply_chat_template to false.
+    VLMDecodedResults generate(
+        const std::string& prompt,
+        const std::vector<ov::Tensor>& images,
+        const std::vector<ov::Tensor>& videos,
         const GenerationConfig& generation_config,
         const StreamerVariant& streamer
     );
@@ -244,7 +263,10 @@ class OPENVINO_GENAI_EXPORTS VLMPipeline {
 /*
  * utils that allow to use generate() in the following way:
  * pipe.generate(prompt, ov::genai::image(image_tensor)).
+ * pipe.generate(prompt, ov::genai::images(image_tensors)).
+ * pipe.generate(prompt, ov::genai::videos(videos_tensors)).
 */
 static constexpr ov::Property<ov::Tensor> image{"image"};
 static constexpr ov::Property<std::vector<ov::Tensor>> images{"images"};
+static constexpr ov::Property<std::vector<ov::Tensor>> videos{"videos"};
 }

src/cpp/src/continuous_batching/pipeline.cpp

@@ -239,10 +239,21 @@ GenerationHandle ContinuousBatchingPipeline::add_request(uint64_t request_id, co
     return m_impl->add_request(request_id, input_ids, sampling_params);
 }
 
-GenerationHandle ContinuousBatchingPipeline::add_request(uint64_t request_id, const std::string& prompt, const std::vector<ov::Tensor>& images, const ov::genai::GenerationConfig& sampling_params) {
+GenerationHandle ContinuousBatchingPipeline::add_request(uint64_t request_id,
+                                                         const std::string& prompt,
+                                                         const std::vector<ov::Tensor>& images,
+                                                         const ov::genai::GenerationConfig& sampling_params) {
     return m_impl->add_request(request_id, prompt, images, sampling_params);
 }
 
+GenerationHandle ContinuousBatchingPipeline::add_request(uint64_t request_id,
+                                                         const std::string& prompt,
+                                                         const std::vector<ov::Tensor>& images,
+                                                         const std::vector<ov::Tensor>& videos,
+                                                         const ov::genai::GenerationConfig& sampling_params) {
+    return m_impl->add_request(request_id, prompt, images, videos, sampling_params);
+}
+
 void ContinuousBatchingPipeline::step() {
     m_impl->step();
 }
@@ -279,6 +290,14 @@ std::vector<VLMDecodedResults> ContinuousBatchingPipeline::generate(
     return m_impl->generate(prompts, images, sampling_params, streamer);
 }
 
+std::vector<VLMDecodedResults> ContinuousBatchingPipeline::generate(
+    const std::vector<std::string>& prompts,
+    const std::vector<std::vector<ov::Tensor>>& images,
+    const std::vector<std::vector<ov::Tensor>>& videos,
+    const std::vector<GenerationConfig>& sampling_params,
+    const StreamerVariant& streamer) {
+    return m_impl->generate(prompts, images, videos, sampling_params, streamer);
+}
 
 void ContinuousBatchingPipeline::start_chat(const std::string& system_message) {
     m_impl->finish_chat();

src/cpp/src/continuous_batching/pipeline_base.cpp

@@ -35,11 +35,14 @@ void ContinuousBatchingPipeline::IContinuousBatchingPipeline::finish_chat() {
     m_is_chat_conversation = false;
     m_history.clear();
     m_history_images.clear();
+    m_history_videos.clear();
     m_history_image_ids.clear();
+    m_history_video_ids.clear();
     if (m_inputs_embedder) {
         m_inputs_embedder->finish_chat();
     }
     m_image_id = 0;
+    m_video_id = 0;
 };
 
 std::vector<GenerationResult>
@@ -151,40 +154,72 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
              const std::vector<std::string>& prompts,
              const std::vector<std::vector<ov::Tensor>>& rgbs_vector,
              const std::vector<GenerationConfig>& sampling_params,
+             const StreamerVariant& streamer) {
+    return generate(prompts, rgbs_vector, {}, sampling_params, streamer);
+}
+
+std::vector<VLMDecodedResults>
+ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
+             const std::vector<std::string>& prompts,
+             const std::vector<std::vector<ov::Tensor>>& rgbs_vector,
+             const std::vector<std::vector<ov::Tensor>>& video_vector,
+             const std::vector<GenerationConfig>& sampling_params,
              const StreamerVariant& streamer)  {
     auto generate_start_time = std::chrono::steady_clock::now();
     OPENVINO_ASSERT(m_model_input_type == ModelInputType::EMBEDDINGS);
 
     OPENVINO_ASSERT(prompts.size() == sampling_params.size(), "Number of prompts should be equal to the number of generation configs.");
-    OPENVINO_ASSERT(prompts.size() == rgbs_vector.size(), "Number of prompts should be equal to the number of images vectors.");
 
     std::vector<ov::Tensor> input_embeds_list;
     std::vector<ov::Tensor> token_type_ids_list;
 
     std::vector<VLMPerfMetrics> vlm_perf_metrics(prompts.size());
     std::vector<EncodedImage> encoded_images = {};
+    std::vector<std::vector<EncodedImage>> encoded_videos = {};
 
     if (m_is_chat_conversation) {
         OPENVINO_ASSERT(1 == prompts.size(), "Can't chat with multiple prompts");
-        const auto& rgbs = rgbs_vector[0];
         const auto& prompt = prompts[0];
         auto start_get_inputs_embeds = std::chrono::steady_clock::now();
-        encoded_images = m_inputs_embedder->encode_images(rgbs);
+
+        auto video_rgbs = video_vector.size() > 0 ? video_vector[0] : std::vector<ov::Tensor>{};
+        for (auto& vd : video_rgbs) {
+            auto encoded_vd = m_inputs_embedder->encode_video({vd});
+            m_history_videos.push_back(encoded_vd);
+            encoded_videos.push_back(encoded_vd);
+        }
+
+        auto image_rgbs = rgbs_vector.size() > 0 ? rgbs_vector[0] : std::vector<ov::Tensor>{};
+        encoded_images = m_inputs_embedder->encode_images(image_rgbs);
         m_history_images.insert(m_history_images.end(), encoded_images.begin(), encoded_images.end());
 
-        const auto [unified_prompt, image_sequence] = m_inputs_embedder->normalize_prompt(prompt, m_image_id, encoded_images);
-        m_history.push_back({{"role", "user"}, {"content", unified_prompt}});
-        m_history_image_ids.insert(m_history_image_ids.end(), image_sequence.begin(), image_sequence.end());
+        auto norm_prompt = m_inputs_embedder->normalize_prompt(prompt, m_image_id, m_video_id, encoded_images, encoded_videos);
+        m_history.push_back({{"role", "user"}, {"content", norm_prompt.unified_prompt}});
+        m_history_image_ids.insert(m_history_image_ids.end(), norm_prompt.images_sequence.begin(), norm_prompt.images_sequence.end());
+        m_history_video_ids.insert(m_history_video_ids.end(), norm_prompt.videos_sequence.begin(), norm_prompt.videos_sequence.end());
 
         std::string templated_history = m_tokenizer.apply_chat_template(m_history, true);
 
         m_inputs_embedder->set_apply_chat_template_status(false);
+
         if (m_inputs_embedder->has_token_type_ids()) {
-            auto [embeds, tt_ids] = m_inputs_embedder->get_inputs_embeds_with_token_type_ids(templated_history, m_history_images, vlm_perf_metrics[0], rgbs.size() > 0, m_history_image_ids);
+            auto [embeds, tt_ids] = m_inputs_embedder->get_inputs_embeds_with_token_type_ids(templated_history,
+                                                                                             m_history_images,
+                                                                                             m_history_videos,
+                                                                                             vlm_perf_metrics[0],
+                                                                                             true,
+                                                                                             m_history_image_ids,
+                                                                                             m_history_video_ids);
             input_embeds_list.push_back(std::move(embeds));
             token_type_ids_list.push_back(std::move(tt_ids));
         } else {
-            input_embeds_list.emplace_back(m_inputs_embedder->get_inputs_embeds(templated_history, m_history_images, vlm_perf_metrics[0], rgbs.size() > 0, m_history_image_ids));
+            input_embeds_list.emplace_back(m_inputs_embedder->get_inputs_embeds(templated_history,
+                                                                                m_history_images,
+                                                                                m_history_videos,
+                                                                                vlm_perf_metrics[0],
+                                                                                true,
+                                                                                m_history_image_ids,
+                                                                                m_history_video_ids));
         }
 
         auto end_get_inputs_embeds = std::chrono::steady_clock::now();
@@ -193,19 +228,41 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
     } else {
         for (size_t i = 0; i < prompts.size(); i++) {
             const auto& prompt = prompts[i];
-            const auto& rgbs = rgbs_vector[i];
+
             auto start_get_inputs_embeds = std::chrono::steady_clock::now();
-            const auto encoded_images = m_inputs_embedder->encode_images(rgbs);
-            auto [unified_prompt, image_sequence] = m_inputs_embedder->normalize_prompt(prompt, m_image_id, encoded_images);
+
+            auto image_rgbs = rgbs_vector.size() > 0 ? rgbs_vector[i] : std::vector<ov::Tensor>{};
+            auto video_rgbs = video_vector.size() > 0 ? video_vector[i] : std::vector<ov::Tensor>{};
+            const auto encoded_images = m_inputs_embedder->encode_images(image_rgbs);
+            std::vector<std::vector<ov::genai::EncodedImage>> encoded_videos;
+            for (auto& vd : video_rgbs) {
+                auto encoded_vd = m_inputs_embedder->encode_video({vd});
+                encoded_videos.push_back(encoded_vd);
+            }
+
+            auto norm_prompt = m_inputs_embedder->normalize_prompt(prompt, m_image_id, m_video_id, encoded_images, encoded_videos);
 
             m_inputs_embedder->set_apply_chat_template_status(sampling_params[i].apply_chat_template);
 
             if (m_inputs_embedder->has_token_type_ids()) {
-                auto [embeds, tt_ids] = m_inputs_embedder->get_inputs_embeds_with_token_type_ids(unified_prompt, encoded_images, vlm_perf_metrics[i], true, image_sequence);
+                auto [embeds, tt_ids] =
+                    m_inputs_embedder->get_inputs_embeds_with_token_type_ids(norm_prompt.unified_prompt,
+                                                                             encoded_images,
+                                                                             encoded_videos,
+                                                                             vlm_perf_metrics[i],
+                                                                             true,
+                                                                             norm_prompt.images_sequence,
+                                                                             norm_prompt.videos_sequence);
                 input_embeds_list.push_back(std::move(embeds));
                 token_type_ids_list.push_back(std::move(tt_ids));
             } else {
-                input_embeds_list.emplace_back(m_inputs_embedder->get_inputs_embeds(unified_prompt, encoded_images, vlm_perf_metrics[i], true, image_sequence));
+                input_embeds_list.emplace_back(m_inputs_embedder->get_inputs_embeds(norm_prompt.unified_prompt,
+                                                                                    encoded_images,
+                                                                                    encoded_videos,
+                                                                                    vlm_perf_metrics[i],
+                                                                                    true,
+                                                                                    norm_prompt.images_sequence,
+                                                                                    norm_prompt.videos_sequence));
             }
 
             auto end_get_inputs_embeds = std::chrono::steady_clock::now();
@@ -240,13 +297,16 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
         m_inputs_embedder->update_chat_history(results[0].texts[0], encoded_results[0].m_status);
         if (encoded_results[0].m_status != ov::genai::GenerationStatus::CANCEL) {
             m_image_id += encoded_images.size();
+            m_video_id += encoded_videos.size();
             m_history.push_back({{"role", "assistant"}, {"content", results[0].texts[0]}});
         }
         else {
             m_history.pop_back();
             for (size_t idx = 0; idx < encoded_images.size(); idx++) {
                 m_history_image_ids.pop_back();
                 m_history_images.pop_back();
+                m_history_video_ids.pop_back();
+                m_history_videos.pop_back();
             }
         }
     }
@@ -264,14 +324,42 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::add_request(uint64_t re
     {
         std::lock_guard<std::mutex> lock(m_embeddings_mutex);
         m_inputs_embedder->set_apply_chat_template_status(sampling_params.apply_chat_template);
-        const auto encoded_images = m_inputs_embedder->encode_images(rgbs);
+
+        auto encoded_images = m_inputs_embedder->encode_images(rgbs);
 
         const auto [unified_prompt, image_sequence] = m_inputs_embedder->normalize_prompt(prompt, 0, encoded_images);
         inputs = m_inputs_embedder->get_inputs_embeds(unified_prompt, encoded_images, metrics, true, image_sequence);
     }
     return add_request(request_id, inputs, sampling_params);
 }
 
+GenerationHandle ContinuousBatchingPipeline::IContinuousBatchingPipeline::add_request(
+    uint64_t request_id,
+    const std::string& prompt,
+    const std::vector<ov::Tensor>& images,
+    const std::vector<ov::Tensor>& videos,
+    GenerationConfig sampling_params) {
+    OPENVINO_ASSERT(m_model_input_type == ModelInputType::EMBEDDINGS, "Model doesn't support embeddings.");
+
+    ov::genai::VLMPerfMetrics metrics;
+    ov::Tensor inputs;
+    {
+        std::lock_guard<std::mutex> lock(m_embeddings_mutex);
+        m_inputs_embedder->set_apply_chat_template_status(sampling_params.apply_chat_template);
+
+        auto encoded_images = m_inputs_embedder->encode_images(images);
+        std::vector<std::vector<ov::genai::EncodedImage>> encoded_videos;
+        for (auto& vd : videos) {
+            auto encoded_vd = m_inputs_embedder->encode_video({vd});
+            encoded_videos.push_back(encoded_vd);
+        }
+
+        auto norm_prompt = m_inputs_embedder->normalize_prompt(prompt, 0, 0, encoded_images, encoded_videos);
+        inputs = m_inputs_embedder->get_inputs_embeds(norm_prompt.unified_prompt, encoded_images, metrics, true, norm_prompt.images_sequence);
+    }
+    return add_request(request_id, inputs, sampling_params);
+}
+
 void ContinuousBatchingPipeline::IContinuousBatchingPipeline::stream_tokens(
     const std::shared_ptr<ThreadedStreamerWrapper>& streamer_ptr,
     const GenerationHandle& handle