Weaviate + Google Vertex AI实现多模态嵌入

多模态嵌入的概述#

多模态嵌入（Multimodal Embeddings）指的是通过对不同数据模式（如文本、图像、音频、视频等）进行嵌入表示。通过这种技术，Weaviate 能够将各种模式的输入（如文本和图像）转化为统一的向量表示，用于高效的相似性搜索或其他机器学习任务。Weaviate 与 Google Vertex AI 的集成使得这一功能变得更加易用和强大。Weaviate 与 Google AI 多模态嵌入的集成使得用户能够处理和搜索不同类型的多模态数据，这对于需要在大规模数据库中处理文本、图像、视频等不同数据模式的场景非常有用。集成的 Google Vertex AI 模型不仅性能强大，还能够支持各种复杂的语义和多模态搜索，提升了数据管理和查询的智能化水平。

使用 Docker 部署 Weaviate 实例#

services:
  weaviate:
    command:
    - --host
    - 0.0.0.0
    - --port
    - '8080'
    - --scheme
    - http
    image: cr.weaviate.io/semitechnologies/weaviate:1.26.4
    ports:
    - 8080:8080
    - 50051:50051
    volumes:
    - ./weaviate_data:/var/lib/weaviate
    - /root/.config/gcloud/application_default_credentials.json:/etc/weaviate/gcp-credentials.json
    restart: on-failure:0
    environment:
      QUERY_DEFAULTS_LIMIT: 25
      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
      DEFAULT_VECTORIZER_MODULE: 'multi2vec-palm'
      ENABLE_MODULES: 'multi2vec-palm,ref2vec-centroid'
      CLUSTER_HOSTNAME: 'node1'
      GOOGLE_APPLICATION_CREDENTIALS: '/etc/weaviate/gcp-credentials.json'
      USE_GOOGLE_AUTH: 'true'

你必须提供合法的 API 凭证才能正确的使用 Vertex AI 集成，具体的配置方式你可以查看API credentials。

编写代码#

连接到 Weaviate 并检查连接#

import weaviate

client = weaviate.connect_to_local()

client.is_ready()

创建 Collection#

from weaviate.classes.config import Configure

if client.collections.exists("AnimeGirls"):
    client.collections.delete("AnimeGirls")

client.collections.create(
    name="AnimeGirls",
    vectorizer_config=Configure.Vectorizer.multi2vec_palm(
        image_fields=["image"],
        text_fields=["text"],
        video_fields=["video"],
        project_id="neurosearch-436306",
        location="europe-west1",
        model_id="multimodalembedding@001",
        dimensions=1408,
    ),
)

创建工具函数#

import base64
def to_base64(file_path: str) -> str:
    with open(file_path, "rb") as file:
        return base64.b64encode(file.read()).decode("utf-8")

导入数据#

import os
from weaviate.util import generate_uuid5
anime_girls = client.collections.get("AnimeGirls")

sources = os.listdir("./images/")

with anime_girls.batch.dynamic() as batch:
    for name in sources:
        print(f"Adding {name}")

        path = "./images/" + name

        batch.add_object(
            {
                "name": name,
                "image": to_base64(path),
                "path": path,
                "mediaType": "image",
            },
            uuid=generate_uuid5(name),
        )

检查所有数据是否导入成功#

if len(anime_girls.batch.failed_objects) > 0:
    print(f"Failed to import {len(anime_girls.batch.failed_objects)} objects")
    for failed_object in anime_girls.batch.failed_objects:
        print(f"e.g. Failed to import object with error: {failed_object.message}")
else:
    print("All objects imported successfully")

通过 Text 检索#

import json
response = anime_girls.query.near_text(
    query="Seeing a girl through glasses",
    return_properties=["name", "path", "mediaType"],
    limit=2,
)

for obj in response.objects:
    print(json.dumps(obj.properties, indent=2))

from IPython.display import Image, display

def display_image(item: dict):
    path = item["path"]
    display(Image(path, width=300))

display_image(response.objects[0].properties)

通过 Image 检索#

response = anime_girls.query.near_image(
    near_image=to_base64("./images/121955436_p0_master1200.jpg"),
    return_properties=["name", "path", "mediaType"],
    limit=2,
)

# for obj in response.objects:
#     print(json.dumps(obj.properties, indent=2))

display_image(response.objects[0].properties)

混合检索#

response = anime_girls.query.hybrid(
    query="Seeing a girl through glasses",
    return_properties=["name", "path", "mediaType"],
    limit=2,
)

# for obj in response.objects:
#     print(json.dumps(obj.properties, indent=2))

display_image(response.objects[0].properties)

返回所有向量#

import numpy as np
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

# 假设embedding是你的1408维数据
embedding = np.array([item.vector['default'] for item in anime_girls.iterator(include_vector=True)])

# 使用PCA将1408维数据降到2维
pca = PCA(n_components=2)
reduced_embedding = pca.fit_transform(embedding)

# 绘制降维后的数据
plt.figure(figsize=(10, 7))
plt.scatter(reduced_embedding[:, 0], reduced_embedding[:, 1], alpha=0.5)
plt.title('PCA of AnimeGirls Embeddings')
plt.xlabel('Principal Component 1')
plt.ylabel('Principal Component 2')
plt.show()