Intel RealSense T265鱼眼相机图像获取并矫正畸变(Unity)

1.添加RsDevice：

2.设置开启鱼眼相机：

3.新建RsStreamTextureRendererFisheye.cs和Fisheye.shader：

using Intel.RealSense;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Threading;
using UnityEngine;
using UnityEngine.Events;public class RsStreamTextureRendererFisheye : MonoBehaviour
{private static TextureFormat Convert(Format lrsFormat){switch (lrsFormat){case Format.Z16: return TextureFormat.R16;case Format.Disparity16: return TextureFormat.R16;case Format.Rgb8: return TextureFormat.RGB24;case Format.Rgba8: return TextureFormat.RGBA32;case Format.Bgra8: return TextureFormat.BGRA32;case Format.Y8: return TextureFormat.Alpha8;case Format.Y16: return TextureFormat.R16;case Format.Raw16: return TextureFormat.R16;case Format.Raw8: return TextureFormat.Alpha8;case Format.Disparity32: return TextureFormat.RFloat;case Format.Yuyv:case Format.Bgr8:case Format.Raw10:case Format.Xyz32f:case Format.Uyvy:case Format.MotionRaw:case Format.MotionXyz32f:case Format.GpioRaw:case Format.Any:default:throw new ArgumentException(string.Format("librealsense format: {0}, is not supported by Unity", lrsFormat));}}private static int BPP(TextureFormat format){switch (format){case TextureFormat.ARGB32:case TextureFormat.BGRA32:case TextureFormat.RGBA32:return 32;case TextureFormat.RGB24:return 24;case TextureFormat.R16:return 16;case TextureFormat.R8:case TextureFormat.Alpha8:return 8;default:throw new ArgumentException("unsupported format {0}", format.ToString());}}public RsFrameProvider Source;[System.Serializable]public class TextureEvent : UnityEvent<Texture> { }public Stream _stream;public Format _format;public int _streamIndex;public FilterMode filterMode = FilterMode.Point;protected Texture2D texture;public ComputeShader shader;public Texture2D uvmap;[Delayed]public float fov = 90;[Space]public TextureEvent textureBinding;FrameQueue q;Predicate<Frame> matcher;void Start(){Source.OnStart += OnStartStreaming;Source.OnStop += OnStopStreaming;}void OnDestroy(){if (texture != null){Destroy(texture);texture = null;}if (q != null){q.Dispose();}}protected void OnStopStreaming(){Source.OnNewSample -= OnNewSample;if (q != null){q.Dispose();q = null;}}public void OnStartStreaming(PipelineProfile activeProfile){q = new FrameQueue(1);matcher = new Predicate<Frame>(Matches);Source.OnNewSample += OnNewSample;}private bool Matches(Frame f){using (var p = f.Profile)return p.Stream == _stream && p.Format == _format && p.Index == _streamIndex;}void OnNewSample(Frame frame){try{if (frame.IsComposite){using (var fs = frame.As<FrameSet>())using (var f = fs.FirstOrDefault(matcher)){if (f != null)q.Enqueue(f);return;}}if (!matcher(frame))return;using (frame){q.Enqueue(frame);}}catch (Exception e){Debug.LogException(e);// throw;}}bool HasTextureConflict(VideoFrame vf){return !texture ||texture.width != vf.Width ||texture.height != vf.Height ||BPP(texture.format) != vf.BitsPerPixel;}protected void LateUpdate(){if (q != null){VideoFrame frame;if (q.PollForFrame<VideoFrame>(out frame))using (frame)ProcessFrame(frame);}}private void ProcessFrame(VideoFrame frame){if (HasTextureConflict(frame)){if (texture != null){Destroy(texture);}using (var p = frame.Profile){bool linear = (QualitySettings.activeColorSpace != ColorSpace.Linear)|| (p.Stream != Stream.Color && p.Stream != Stream.Infrared);texture = new Texture2D(frame.Width, frame.Height, Convert(p.Format), false, linear){wrapMode = TextureWrapMode.Clamp,filterMode = filterMode};}textureBinding.Invoke(texture);}texture.LoadRawTextureData(frame.Data, frame.Stride * frame.Height);texture.Apply();if (uvmap == null){CreateUvMap(frame);Shader.SetGlobalTexture("_UVMap", uvmap);}}static Vector2Int Undistort(float x, float y, ref Intrinsics intrin, int W, int H, float fov){var uv = new Vector2(x / W * intrin.width, y / H * intrin.height);// see https://github.com/IntelRealSense/librealsense/blob/master/include/librealsense2/rsutil.huv.x = (uv.x - intrin.ppx) / intrin.fx;uv.y = (uv.y - intrin.ppy) / intrin.fy;float rd = uv.magnitude;float theta = rd;float theta2 = rd * rd;for (int i = 0; i < 4; i++){float f = theta * (1 + theta2 * (intrin.coeffs[0] + theta2 * (intrin.coeffs[1] + theta2 * (intrin.coeffs[2] + theta2 * intrin.coeffs[3])))) - rd;float df = 1 + theta2 * (3 * intrin.coeffs[0] + theta2 * (5 * intrin.coeffs[1] + theta2 * (7 * intrin.coeffs[2] + 9 * theta2 * intrin.coeffs[3])));theta -= f / df;theta2 = theta * theta;}float r = Mathf.Tan(theta);uv *= r / rd;// see https://github.com/IntelRealSense/librealsense/blob/master/wrappers/python/examples/t265_stereo.pyfloat stereo_fov_rad = fov * (Mathf.PI / 180);float stereo_height_px = H;float stereo_focal_px = stereo_height_px / 2 / Mathf.Tan(stereo_fov_rad / 2);uv = uv * stereo_focal_px + Vector2.one * (stereo_height_px - 1f) / 2f;uv += Vector2.one * 0.5f;return Vector2Int.RoundToInt(uv);}private void CreateUvMap(VideoFrame frame){const int W = 848, H = 800;if (uvmap)DestroyImmediate(uvmap);uvmap = new Texture2D(W, H, TextureFormat.RGFloat, false, true){filterMode = FilterMode.Point,wrapMode = TextureWrapMode.Clamp};using (var profile = frame.Profile.As<VideoStreamProfile>()){var intrinsics = profile.GetIntrinsics();Debug.Log(intrinsics);var uvs = new Vector2[H, W];// see https://github.com/IntelRealSense/librealsense/blob/master/src/proc/align.cppfor (int y = 0; y < H; ++y){for (int x = 0; x < W; ++x){var uv = Undistort(x - 0.5f, y - 0.5f, ref intrinsics, W, H, fov);var uv1 = Undistort(x + 0.5f, y + 0.5f, ref intrinsics, W, H, fov);if (uv.x < 0 || uv.y < 0 || uv1.x >= W || uv1.y >= H)continue;float u = (float)x / W;float v = (float)y / H;for (int j = uv.y; j <= uv1.y; ++j){for (int i = uv.x; i <= uv1.x; ++i){uvs[j, i].Set(u, v);}}}}var h = GCHandle.Alloc(uvs, GCHandleType.Pinned);uvmap.LoadRawTextureData(h.AddrOfPinnedObject(), W * H * 2 * sizeof(float));uvmap.Apply();h.Free();}}
}

Fisheye.shader：

Shader "Custom/Fisheye" {Properties {_MainTex ("MainTex", 2D) = "white" {}_Gamma ("Gamma", float) = 0.45}SubShader {Tags { "QUEUE"="Transparent" "IGNOREPROJECTOR"="true" "RenderType"="Transparent" "PreviewType"="Plane" }Pass {     ZWrite OffCull OffFog { Mode Off }ColorMask RGBCGPROGRAM#pragma vertex vert_img#pragma fragment frag#pragma target 3.0#pragma glsl#include "UnityCG.cginc"sampler2D _MainTex;uniform sampler2D _UVMap;float _Gamma;half4 frag (v2f_img pix) : SV_Target{float2 uv = pix.uv;float2 uvmap = tex2D(_UVMap, uv).xy;             float Y = pow(tex2D(_MainTex, uvmap).a, _Gamma);return float4(Y.rrr, 1);}ENDCG}} FallBack Off
}

4.新建Raw Image以便显示相机图像：

设置材质球和着色器：

5.绑定相机：

这样，就可以显示正常相机图像，下面看看矫正前后对比：

矫正前：

矫正后：

Intel RealSense T265鱼眼相机图像获取并矫正畸变(Unity)相关推荐

Intel Realsense T265使用教程
Intel Realsense T265使用教程 0 更新日志 1 T265参数 1.1 硬件参数 1.2 坐标系描述 2 T265 数据读取 2.1 环境安装 2.2 读取T265内外参数信息 2. ...
Intel RealSense D435i 深度相机介绍
参考: https://www.sohu.com/a/340984033_715754 https://www.chiphell.com/thread-1945054-1-1.html https:/ ...
Intel RealSense D455 深度相机
2020年6月17日发布的相机:Intel RealSense D455 是 Intel RealSense D435i 的升级版. 外观与D435i基本一样,都包含IMU,摄像头具体参数未知,适用范 ...
intel realsense D400系列相机介绍（一）
文章目录 intel realsense D4 2 intel realsense D400介绍 2.3 立体视觉深度技术概述 2.4 摄像机系统框图 2.5 英特尔Intel® RealSense™ ...
BundleFusion使用Intel RealSense D435i深度相机实现
BundleFusion是目前效果最好的实时稠密三维重建项目,它支持多种类别的深度相机,本次使用英特尔的RealSense D435i深度相机来采集颜色图和深度图,配置的方式如下: 一.环境与资源准备 ...
ubuntu18.04 Intel Realsense T265与Realsense D435i 使用教程
主要包括: realsense sdk驱动安装与ros包安装编译 D435i与t256相机使用多个相机联合使用官网链接: https://github.com/IntelRealSense/rea ...
「完结撒花」使用intel realsense D435i深度相机获取信息，DOBOT MG400机械手实现动态实时抓取，并做了个GUI界面（python实现）
GUI界面如下,丑是丑了点,但很好用,嘿嘿. GUI部分 1.GUI界面可以实现的功能暂停・继续相机画面显示彩色RGB图像・深度图像・开运算结果图・进行图像处理后的图片(右侧) 调节opencv轮 ...
Realsense T265双目+IMU传感器追踪相机的环境配置指南（Ubuntu+Windows）
T265追踪相机,可以直接读取里程计信息,直接输出位置.速度等参数,为了了解如何使用,利用网上的信息进行了环境的配置,先测试的是Windows平台的使用,后来在Ubuntu下面配置环境,最后根据历程代 ...
Windows下载安装Intel RealSense SDK2.0
下载地址: https://github.com/IntelRealSense/librealsense/releases 选择下载Intel.RealSense.SDK-WIN10-2.50.0.3 ...

Intel RealSense T265鱼眼相机图像获取并矫正畸变(Unity)

Intel RealSense T265鱼眼相机图像获取并矫正畸变(Unity)相关推荐

最新文章

热门文章