可视化教程

介绍 

本教程介绍如何使用Zivid SDK和第三方库来可视化Zivid相机捕获的3D和2D数据。

先决条件

安装 Zivid 软件。
对于Python：安装 Zivid-python

本教程从Zivid帧（frame）开始。查看捕获教程了解更多有关如何捕获帧的信息。

点云 

在C++和C#中使用Zivid SDK 

您可以通过帧（frame）可视化点云。

C++

std::cout << "Setting up visualization" << std::endl;
Zivid::Visualization::Visualizer visualizer;

std::cout << "Visualizing point cloud" << std::endl;
visualizer.showMaximized();
visualizer.show(frame);
visualizer.resetToFit();

std::cout << "Running visualizer. Blocking until window closes." << std::endl;
visualizer.run();

C＃

Console.WriteLine("Setting up visualization");
using (var visualizer = new Zivid.NET.Visualization.Visualizer())
{
    Console.WriteLine("Visualizing point cloud");
    visualizer.Show(frame);
    visualizer.ShowMaximized();
    visualizer.ResetToFit();

    Console.WriteLine("Running visualizer. Blocking until window closes.");
    visualizer.Run();
}

Python

print("Visualizing point cloud")
with zivid.visualization.Visualizer() as visualizer:
    visualizer.set_window_title("Zivid Point Cloud Visualizer")
    visualizer.colors_enabled = True
    visualizer.axis_indicator_enabled = True
    visualizer.show(frame)
    visualizer.reset_to_fit()
    visualizer.run()

您也可以从点云对象来可视化点云。

C++

std::cout << "Getting point cloud from frame" << std::endl;
auto pointCloud = frame.pointCloud();

std::cout << "Setting up visualization" << std::endl;
Zivid::Visualization::Visualizer visualizer;

std::cout << "Visualizing point cloud" << std::endl;
visualizer.showMaximized();
visualizer.show(pointCloud);
visualizer.resetToFit();

std::cout << "Running visualizer. Blocking until window closes." << std::endl;
visualizer.run();

C＃

Console.WriteLine("Getting point cloud from frame");
var pointCloud = frame.PointCloud;

Console.WriteLine("Setting up visualization");
var visualizer = new Zivid.NET.Visualization.Visualizer();

Console.WriteLine("Visualizing point cloud");
visualizer.Show(pointCloud);
visualizer.ShowMaximized();
visualizer.ResetToFit();

Console.WriteLine("Running visualizer. Blocking until window closes.");
visualizer.Run();

Python

    point_cloud = frame.point_cloud()

with zivid.visualization.Visualizer() as visualizer:
    visualizer.set_window_title("Zivid Point Cloud Visualizer")
    visualizer.colors_enabled = True
    visualizer.axis_indicator_enabled = True
    visualizer.show(point_cloud)
    visualizer.reset_to_fit()
    visualizer.run()

Live 3D Point Cloud 

The Zivid SDK can be used to continuously capture and visualize 3D point clouds in a loop.

Zivid SDK in C++

The visualizer runs in a dedicated thread while the main thread continuously captures new frames and pushes them to the visualizer.

C++

std::cout << "Setting up visualization" << std::endl;
std::atomic_bool visualizerRunning{ false };
auto visualizerPromise = std::promise<Zivid::Visualization::Visualizer *>();
auto visualizerFuture = visualizerPromise.get_future();

std::thread visualizationThread([&frame, &visualizerPromise, &visualizerRunning]() {
    auto visualizer = Zivid::Visualization::Visualizer();

    // Pass the visualizer to the main thread
    visualizerPromise.set_value(&visualizer);

    std::cout << "Visualizing point cloud" << std::endl;
    visualizer.showMaximized();
    visualizer.show(frame);
    visualizer.resetToFit();

    std::cout << "Running visualizer. Blocking until window closes." << std::endl;
    visualizerRunning = true;
    visualizer.run();
    visualizerRunning = false;
});

// Get the visualizer handle in the main thread
auto visualizerHandle = visualizerFuture.get();

while(!visualizerRunning)
{
    std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
while(visualizerRunning)
{
    frame = camera.capture2D3D(settings);
    if(!visualizerRunning)
    {
        break;
    }
    visualizerHandle->show(frame);
    std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
visualizationThread.join();
std::cout << "Visualizer closed" << std::endl;

C＃

Console.WriteLine("Setting up visualization");
var visualizerRunning = new ManualResetEventSlim(false);
Zivid.NET.Visualization.Visualizer visualizerHandle = null;
var visualizerReady = new ManualResetEventSlim(false);

var visualizationThread = new Thread(() =>
{
    using (var visualizer = new Zivid.NET.Visualization.Visualizer())
    {
        // Pass the visualizer to the main thread
        visualizerHandle = visualizer;
        visualizerReady.Set();

        Console.WriteLine("Visualizing point cloud");
        visualizer.ShowMaximized();
        visualizer.Show(frame);
        visualizer.ResetToFit();

        Console.WriteLine("Running visualizer. Blocking until window closes.");
        visualizerRunning.Set();
        visualizer.Run();
        visualizerRunning.Reset();
    }
});
visualizationThread.Start();

// Get the visualizer handle in the main thread
visualizerReady.Wait();

visualizerRunning.Wait();
while (visualizerRunning.IsSet)
{
    frame = camera.Capture2D3D(settings);
    if (!visualizerRunning.IsSet)
        break;
    visualizerHandle.Show(frame);
    Thread.Sleep(10);
}
visualizationThread.Join();
Console.WriteLine("Visualizer closed");

Python

print("Setting up visualization")
visualizer_running = threading.Event()

print("Visualizing point cloud")
with zivid.visualization.Visualizer() as visualizer:
    visualizer.show(frame)
    visualizer.reset_to_fit()

    def _capture_thread() -> None:
        while visualizer_running.is_set():
            new_frame = camera.capture_2d_3d(settings)
            if visualizer_running.is_set():
                visualizer.show(new_frame)
            time.sleep(0.01)

    capture_thread = threading.Thread(target=_capture_thread)

    print("Running visualizer. Blocking until window closes.")
    visualizer_running.set()
    capture_thread.start()
    visualizer.run()
    visualizer_running.clear()

    capture_thread.join()

print("Visualizer closed")

To exit the loop by pressing 'q' in the terminal:

C++

std::cout << "Setting up visualization" << std::endl;
std::atomic_bool visualizerRunning{ false };
std::atomic_bool acceptEnd{ true };
std::atomic_bool quitRequested{ false };
auto visualizerPromise = std::promise<Zivid::Visualization::Visualizer *>();
auto visualizerFuture = visualizerPromise.get_future();

std::thread visualizationThread([&frame, &visualizerPromise, &visualizerRunning, &acceptEnd, &quitRequested]() {
    auto visualizer = Zivid::Visualization::Visualizer();

    // Pass the visualizer to the main thread
    visualizerPromise.set_value(&visualizer);

    std::cout << "Visualizing point cloud" << std::endl;
    visualizer.showMaximized();
    visualizer.show(frame);
    visualizer.resetToFit();

    std::cout << "Running visualizer. Blocking until window closes." << std::endl;
    do
    {
        visualizerRunning = true;
        visualizer.run();
        visualizerRunning = false;

        if(quitRequested)
        {
            break;
        }
        std::cout << "Visualizer window closed by user. It will be reopened if we're currently capturing."
                  << std::endl;

    } while(!acceptEnd);
});

// Get the visualizer handle in the main thread
auto visualizerHandle = visualizerFuture.get();

while(!visualizerRunning)
{
    std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
std::cout << "Press 'q' in the terminal to quit " << std::endl;
while(visualizerRunning)
{
    int key = getKeyNonBlocking();
    if(key == 'q')
    {
        std::cout << "Closing application because user pressed 'q'" << std::endl;
        quitRequested = true;
        visualizerHandle->close();
    }
    else
    {
        acceptEnd = false;
        frame = camera.capture2D3D(settings);
        visualizerHandle->show(frame);
        acceptEnd = true;
    }
    std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
visualizationThread.join();
std::cout << "Visualizer closed" << std::endl;

C＃

Console.WriteLine("Setting up visualization");
var visualizerRunning = new ManualResetEventSlim(false);
var acceptEnd = new ManualResetEventSlim(true);
var quitRequested = new ManualResetEventSlim(false);
Zivid.NET.Visualization.Visualizer visualizerHandle = null;
var visualizerReady = new ManualResetEventSlim(false);

var visualizationThread = new Thread(() =>
{
    using (var visualizer = new Zivid.NET.Visualization.Visualizer())
    {
        // Pass the visualizer to the main thread
        visualizerHandle = visualizer;
        visualizerReady.Set();

        Console.WriteLine("Visualizing point cloud");
        visualizer.ShowMaximized();
        visualizer.Show(frame);
        visualizer.ResetToFit();

        Console.WriteLine("Running visualizer. Blocking until window closes.");
        do
        {
            visualizerRunning.Set();
            visualizer.Run();
            visualizerRunning.Reset();

            if (quitRequested.IsSet)
                break;
            Console.WriteLine(
                "Visualizer window closed by user. It will be reopened if we're currently capturing."
            );
        } while (!acceptEnd.IsSet);
    }
});
visualizationThread.Start();

// Get the visualizer handle in the main thread
visualizerReady.Wait();

visualizerRunning.Wait();
Console.WriteLine("Press 'q' in the terminal to quit");
while (visualizerRunning.IsSet)
{
    if (Console.KeyAvailable)
    {
        var key = Console.ReadKey(intercept: true);
        if (key.KeyChar == 'q')
        {
            Console.WriteLine("Closing application because user pressed 'q'");
            quitRequested.Set();
            Application.DoEvents();
            foreach (Form form in Application.OpenForms)
            {
                if (form.Text.Contains("Zivid"))
                {
                    form.Invoke(new Action(() => form.Close()));
                    break;
                }
            }
        }
    }
    else
    {
        acceptEnd.Reset();
        frame = camera.Capture2D3D(settings);
        visualizerHandle.Show(frame);
        acceptEnd.Set();
    }
    Thread.Sleep(10);
}
visualizationThread.Join();
Console.WriteLine("Visualizer closed");

Python

print("Setting up visualization")
visualizer_running = threading.Event()
accept_end = threading.Event()
accept_end.set()
quit_requested = threading.Event()

use_raw_terminal = sys.platform != "win32" and sys.stdin.isatty()
if use_raw_terminal:
    old_terminal_settings = termios.tcgetattr(sys.stdin)
    tty.setcbreak(sys.stdin.fileno())
try:
    print("Visualizing point cloud")
    with zivid.visualization.Visualizer() as visualizer:
        visualizer.show(frame)
        visualizer.reset_to_fit()

        def _capture_and_keypress_thread() -> None:
            print("Press 'q' in the terminal to quit")
            while not quit_requested.is_set():
                if not visualizer_running.wait(timeout=0.01):
                    continue
                key = _get_key_non_blocking()
                if key == "q":
                    print("Closing application because user pressed 'q'")
                    quit_requested.set()
                    visualizer.close()
                else:
                    accept_end.clear()
                    new_frame = camera.capture_2d_3d(settings)
                    if visualizer_running.is_set():
                        visualizer.show(new_frame)
                    accept_end.set()
                time.sleep(0.01)

        capture_thread = threading.Thread(target=_capture_and_keypress_thread)
        capture_thread.start()

        print("Running visualizer. Blocking until window closes.")
        while True:
            visualizer_running.set()
            visualizer.run()
            visualizer_running.clear()

            if quit_requested.is_set():
                break
            print("Visualizer window closed by user. It will be reopened if we're currently capturing.")
            if accept_end.is_set():
                break

        capture_thread.join()

    print("Visualizer closed")
finally:
    if use_raw_terminal:
        termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_terminal_settings)

彩色图像 

由于Zivid SDK 和 Zivid-Python 不支持2D彩色图像可视化，我们使用第三方库实现该功能：使用 OpenCV 在C++和Python中实现，以及使用 Matplotlib 在Python中实现。

OpenCV in C++ and Python 

首先，我们将点云转换为OpenCV彩色图像。

C++

std::cout << "Converting point cloud to BGRA image in OpenCV format" << std::endl;
cv::Mat bgra = pointCloudToCvBGRA_SRGB(pointCloud);

Python

print("Converting to BGR image in OpenCV format")
bgr = _point_cloud_to_cv_bgr(point_cloud)

下面展示了如何实现将点云转换为彩色图像的功能。

C++

cv::Mat pointCloudToCvBGRA_SRGB(const Zivid::PointCloud &pointCloud)

{
    auto bgra = cv::Mat(pointCloud.height(), pointCloud.width(), CV_8UC4);
    pointCloud.copyData(&(*bgra.begin<Zivid::ColorBGRA_SRGB>()));

    return bgra;
}

Python

def _point_cloud_to_cv_bgr(point_cloud: zivid.PointCloud) -> np.ndarray:
    """Get bgr image from frame.

    Args:
        point_cloud: Zivid point cloud

    Returns:
        bgr: BGR image (HxWx3 ndarray)

    """
    bgra = point_cloud.copy_data("bgra_srgb")

    return bgra[:, :, :3]

小技巧

也可以直接从Zivid 2D彩色图像中获取OpenCV彩色图像。

我们现在可以可视化彩色图像了。

C++

cv::namedWindow("BGR image", cv::WINDOW_AUTOSIZE);
cv::imshow("BGR image", bgra);
cv::waitKey(CI_WAITKEY_TIMEOUT_IN_MS);

Python

cv2.imshow("BGR image", bgr)
print("Press any key to continue")
cv2.waitKey(0)

Python中通过Matplotlib实现 

Matplotlib提供了一种更简单的在Python中可视化彩色图像的方法。

Python

display_rgb(rgba[:, :, 0:3], block=False)

函数实现如下所示。

Python

def display_rgb(rgb: np.ndarray, title: str = "RGB image", block: bool = True) -> None:
    """Display RGB image.

    Args:
        rgb: RGB image (HxWx3 ndarray)
        title: Image title
        block: Stops the running program until the windows is closed

    """
    plt.figure()
    plt.imshow(rgb)
    plt.title(title)
    plt.show(block=block)

深度图 

由于Zivid SDK 和 Zivid-Python 不支持深度图的可视化功能，我们使用第三方库实现该功能：使用 OpenCV 在C++和Python中实现，以及使用 Matplotlib 在Python中实现。

OpenCV in C++ and Python 

首先，我们将点云转换为OpenCV深度图。

C++

std::cout << "Converting to Depth map in OpenCV format" << std::endl;
cv::Mat zColorMap = pointCloudToCvZ(pointCloud);

Python

print("Converting to Depth map in OpenCV format")
z_color_map = _point_cloud_to_cv_z(point_cloud)

下面展示了将点云转换为深度图的函数。

C++

cv::Mat pointCloudToCvZ(const Zivid::PointCloud &pointCloud)
{
    cv::Mat z(pointCloud.height(), pointCloud.width(), CV_8UC1, cv::Scalar(0)); // NOLINT(hicpp-signed-bitwise)
    const auto points = pointCloud.copyPointsZ();

    // Getting min and max values for X, Y, Z images
    const auto *maxZ = std::max_element(points.begin(), points.end(), isLesserOrNan);
    const auto *minZ = std::max_element(points.begin(), points.end(), isGreaterOrNaN);

    // Filling in OpenCV matrix with the cloud data
    for(size_t i = 0; i < pointCloud.height(); i++)
    {
        for(size_t j = 0; j < pointCloud.width(); j++)
        {
            if(std::isnan(points(i, j).z))
            {
                z.at<uchar>(i, j) = 0;
            }
            else
            {
                z.at<uchar>(i, j) =
                    static_cast<unsigned char>((255.0F * (points(i, j).z - minZ->z) / (maxZ->z - minZ->z)));
            }
        }
    }

    // Applying color map
    cv::Mat zColorMap;
    cv::applyColorMap(z, zColorMap, cv::COLORMAP_VIRIDIS);

    // Setting invalid points (nan) to black
    for(size_t i = 0; i < pointCloud.height(); i++)
    {
        for(size_t j = 0; j < pointCloud.width(); j++)
        {
            if(std::isnan(points(i, j).z))
            {
                auto &zRGB = zColorMap.at<cv::Vec3b>(i, j);
                zRGB[0] = 0;
                zRGB[1] = 0;
                zRGB[2] = 0;
            }
        }
    }

    return zColorMap;
}

Python

def _point_cloud_to_cv_z(point_cloud: zivid.PointCloud) -> np.ndarray:
    """Get depth map from frame.

    Args:
        point_cloud: Zivid point cloud

    Returns:
        depth_map_color_map: Depth map (HxWx1 ndarray)

    """
    depth_map = point_cloud.copy_data("z")
    depth_map_uint8 = ((depth_map - np.nanmin(depth_map)) / (np.nanmax(depth_map) - np.nanmin(depth_map)) * 255).astype(
        np.uint8
    )

    depth_map_color_map = cv2.applyColorMap(depth_map_uint8, cv2.COLORMAP_VIRIDIS)

    # Setting nans to black
    depth_map_color_map[np.isnan(depth_map)[:, :]] = 0

    return depth_map_color_map

我们现在可以可视化深度图了。

C++

cv::namedWindow("Depth map", cv::WINDOW_AUTOSIZE);
cv::imshow("Depth map", zColorMap);
cv::waitKey(CI_WAITKEY_TIMEOUT_IN_MS);

Python

cv2.imshow("Depth map", z_color_map)
print("Press any key to continue")
cv2.waitKey(0)

Python中通过Matplotlib实现 

Matplotlib提供了一种更简单的在Python中可视化深度图的方法。

Python

display_depthmap(xyz, block=True)

函数实现如下所示。

Python

def display_depthmap(xyz: np.ndarray, block: bool = True) -> None:
    """Create and display depthmap.

    Args:
        xyz: A numpy array of X, Y and Z point cloud coordinates
        block: Stops the running program until the windows is closed

    """
    plt.figure()
    plt.imshow(
        xyz[:, :, 2],
        vmin=np.nanmin(xyz[:, :, 2]),
        vmax=np.nanmax(xyz[:, :, 2]),
        cmap="viridis",
    )
    plt.colorbar()
    plt.title("Depth map")
    plt.show(block=block)

法线 

由于Zivid SDK不支持法线的可视化，我们使用第三方库实现该功能：使用 PCL 在C++中实现，以及使用 Open3D 在Python中实现。

C++中的使用PCL实现 

我们可以按下面的方式可视化法线。

C++

std::cout << "Visualizing normals" << std::endl;
visualizePointCloudAndNormalsPCL(pointCloudPCL.makeShared(), pointCloudWithNormalsPCL.makeShared());

函数实现如下所示。

C++

void visualizePointCloudAndNormalsPCL(
    const pcl::PointCloud<pcl::PointXYZRGB>::ConstPtr &pointCloud,
    const pcl::PointCloud<pcl::PointXYZRGBNormal>::ConstPtr &pointCloudWithNormals)
{
    auto viewer = pcl::visualization::PCLVisualizer("Viewer");

    int viewRgb(0);
    viewer.createViewPort(0.0, 0.0, 0.5, 1.0, viewRgb);
    viewer.addText("Cloud RGB", 0, 0, "RGBText", viewRgb);
    viewer.addPointCloud<pcl::PointXYZRGB>(pointCloud, "cloud", viewRgb);

    const int normalsSkipped = 10;
    std::cout << "Note! 1 out of " << normalsSkipped << " normals are visualized" << std::endl;

    int viewNormals(0);
    viewer.createViewPort(0.5, 0.0, 1.0, 1.0, viewNormals);
    viewer.addText("Cloud Normals", 0, 0, "NormalsText", viewNormals);
    viewer.addPointCloud<pcl::PointXYZRGBNormal>(pointCloudWithNormals, "cloudNormals", viewNormals);
    viewer.addPointCloudNormals<pcl::PointXYZRGBNormal>(
        pointCloudWithNormals, normalsSkipped, 1, "normals", viewNormals);

    viewer.setCameraPosition(0, 0, -100, 0, 0, 1000, 0, -1, 0);

    std::cout << "Press r to centre and zoom the viewer so that the entire cloud is visible" << std::endl;
    std::cout << "Press q to exit the viewer application" << std::endl;
    while(!viewer.wasStopped())
    {
        viewer.spinOnce(100);
        std::this_thread::sleep_for(std::chrono::milliseconds(100));
    }
}

在Python中使用Open3D 

我们可以按下面的方式可视化法线。

Python

normals_colormap = 0.5 * (1 - normals)
normals_colormap = np.nan_to_num(normals_colormap, nan=0.0, posinf=1.0, neginf=0.0)

print("Visualizing normals in 2D")
display_rgb(rgb=rgba[:, :, :3], title="RGB image", block=False)
display_rgb(rgb=normals_colormap, title="Colormapped normals", block=True)

print("Visualizing normals in 3D")
display_pointcloud_with_downsampled_normals(point_cloud, zivid.PointCloud.Downsampling.by4x4)

函数实现如下所示。

Python

def _copy_to_open3d_point_cloud(xyz: np.ndarray, rgb: np.ndarray, normals: np.ndarray) -> o3d.geometry.PointCloud:
    """Copy point cloud data to Open3D PointCloud object.

    Args:
        xyz: A numpy array of X, Y and Z point cloud coordinates
        rgb: RGB image
        normals: Ordered array of normal vectors, mapped to xyz

    Returns:
        An Open3D PointCloud object
    """
    xyz = np.nan_to_num(xyz).reshape(-1, 3)
    normals = np.nan_to_num(normals).reshape(-1, 3)
    rgb = rgb.reshape(-1, rgb.shape[-1])[:, :3]

    open3d_point_cloud = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(xyz))
    open3d_point_cloud.colors = o3d.utility.Vector3dVector(rgb / 255)
    open3d_point_cloud.normals = o3d.utility.Vector3dVector(normals)

    return open3d_point_cloud


def _display_open3d_point_cloud(open3d_point_cloud: o3d.geometry.PointCloud) -> None:
    """Display Open3D PointCloud object.

    Args:
        open3d_point_cloud: Open3D PointCloud object to display
    """
    visualizer = o3d.visualization.Visualizer()  # pylint: disable=no-member
    visualizer.create_window()
    visualizer.add_geometry(open3d_point_cloud)

    if len(open3d_point_cloud.normals) > 0:
        print("Open 3D controls:")
        print("  n: for normals")
        print("  9: for point cloud colored by normals")
        print("  h: for all controls")
    visualizer.get_render_option().background_color = (0, 0, 0)
    visualizer.get_render_option().point_size = 2
    visualizer.get_render_option().show_coordinate_frame = True
    visualizer.get_view_control().set_front([0, 0, -1])
    visualizer.get_view_control().set_up([0, -1, 0])

    visualizer.run()
    visualizer.destroy_window()


def display_pointcloud_with_downsampled_normals(
    point_cloud: zivid.PointCloud,
    downsampling: zivid.PointCloud.Downsampling,
) -> None:
    """Display point cloud with downsampled normals.

    Args:
        point_cloud: A Zivid point cloud handle
        downsampling: A valid Zivid downsampling factor to apply to normals

    """
    point_cloud.downsample(downsampling)
    rgb = point_cloud.copy_data("rgba_srgb")[:, :, :3]
    xyz = point_cloud.copy_data("xyz")
    normals = point_cloud.copy_data("normals")

    open3d_point_cloud = _copy_to_open3d_point_cloud(xyz, rgb, normals)
    _display_open3d_point_cloud(open3d_point_cloud)

结论 

本教程展示了如何使用Zivid SDK在C++和C#中可视化点云，以及如何使用第三方库在Python中实现可视化功能。还展示了如何使用第三方库在C++、C#和Python中可视化彩色图像、深度图和法线。